2024-11-19 05:23:52,286 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 05:23:52,298 main DEBUG Took 0.009696 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 05:23:52,299 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 05:23:52,299 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 05:23:52,300 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 05:23:52,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,310 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 05:23:52,326 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,327 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,328 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,328 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,329 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,330 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,331 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,331 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,332 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,332 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,333 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,334 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,334 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,335 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,335 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,336 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,336 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,337 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,337 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,338 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,338 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,339 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,339 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 05:23:52,340 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,340 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 05:23:52,342 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 05:23:52,344 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 05:23:52,347 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 05:23:52,347 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 05:23:52,349 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 05:23:52,350 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 05:23:52,359 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 05:23:52,362 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 05:23:52,363 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 05:23:52,364 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 05:23:52,364 main DEBUG createAppenders(={Console}) 2024-11-19 05:23:52,365 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-19 05:23:52,365 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-19 05:23:52,366 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-19 05:23:52,367 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 05:23:52,367 main DEBUG OutputStream closed 2024-11-19 05:23:52,367 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 05:23:52,368 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 05:23:52,368 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-19 05:23:52,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 05:23:52,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 05:23:52,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 05:23:52,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 05:23:52,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 05:23:52,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 05:23:52,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 05:23:52,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 05:23:52,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 05:23:52,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 05:23:52,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 05:23:52,443 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 05:23:52,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 05:23:52,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 05:23:52,444 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 05:23:52,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 05:23:52,445 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 05:23:52,446 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 05:23:52,448 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 05:23:52,448 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-19 05:23:52,449 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 05:23:52,449 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-19T05:23:52,719 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90 2024-11-19 05:23:52,723 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 05:23:52,723 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T05:23:52,734 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-19T05:23:52,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-19T05:23:52,784 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8, deleteOnExit=true 2024-11-19T05:23:52,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-19T05:23:52,786 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/test.cache.data in system properties and HBase conf 2024-11-19T05:23:52,787 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.tmp.dir in system properties and HBase conf 2024-11-19T05:23:52,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.log.dir in system properties and HBase conf 2024-11-19T05:23:52,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-19T05:23:52,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-19T05:23:52,790 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-19T05:23:52,898 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T05:23:53,016 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T05:23:53,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-19T05:23:53,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-19T05:23:53,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-19T05:23:53,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T05:23:53,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-19T05:23:53,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-19T05:23:53,026 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-19T05:23:53,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T05:23:53,027 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-19T05:23:53,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/nfs.dump.dir in system properties and HBase conf 2024-11-19T05:23:53,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/java.io.tmpdir in system properties and HBase conf 2024-11-19T05:23:53,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-19T05:23:53,030 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-19T05:23:53,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-19T05:23:54,359 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-19T05:23:54,472 INFO [Time-limited test {}] log.Log(170): Logging initialized @3080ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-19T05:23:54,560 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T05:23:54,639 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T05:23:54,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T05:23:54,677 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T05:23:54,679 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-19T05:23:54,697 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T05:23:54,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.log.dir/,AVAILABLE} 2024-11-19T05:23:54,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T05:23:54,900 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/java.io.tmpdir/jetty-localhost-41511-hadoop-hdfs-3_4_1-tests_jar-_-any-15455743113913497314/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T05:23:54,907 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41511} 2024-11-19T05:23:54,908 INFO [Time-limited test {}] server.Server(415): Started @3517ms 2024-11-19T05:23:55,564 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-19T05:23:55,573 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-19T05:23:55,574 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-19T05:23:55,575 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-19T05:23:55,575 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-19T05:23:55,576 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.log.dir/,AVAILABLE} 2024-11-19T05:23:55,577 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-19T05:23:55,691 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/java.io.tmpdir/jetty-localhost-46745-hadoop-hdfs-3_4_1-tests_jar-_-any-3400907760808023182/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T05:23:55,692 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:46745} 2024-11-19T05:23:55,693 INFO [Time-limited test {}] server.Server(415): Started @4302ms 2024-11-19T05:23:55,757 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-19T05:23:57,010 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data1/current/BP-23751114-172.17.0.2-1731993833712/current, will proceed with Du for space computation calculation, 2024-11-19T05:23:57,010 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data2/current/BP-23751114-172.17.0.2-1731993833712/current, will proceed with Du for space computation calculation, 2024-11-19T05:23:57,036 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-19T05:23:57,081 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1fb3ca2ed791a3c with lease ID 0xbf195cb27ed6189b: Processing first storage report for DS-2c89d28b-24fb-4551-a99d-6fed4db29f11 from datanode DatanodeRegistration(127.0.0.1:33169, datanodeUuid=101fec1c-8968-4899-8bf8-b796156e53d3, infoPort=40819, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=1725277916;c=1731993833712) 2024-11-19T05:23:57,082 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1fb3ca2ed791a3c with lease ID 0xbf195cb27ed6189b: from storage DS-2c89d28b-24fb-4551-a99d-6fed4db29f11 node DatanodeRegistration(127.0.0.1:33169, datanodeUuid=101fec1c-8968-4899-8bf8-b796156e53d3, infoPort=40819, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=1725277916;c=1731993833712), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-19T05:23:57,083 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf1fb3ca2ed791a3c with lease ID 0xbf195cb27ed6189b: Processing first storage report for DS-bdee27fb-08e5-40cc-b6ff-75a470d54fc7 from datanode DatanodeRegistration(127.0.0.1:33169, datanodeUuid=101fec1c-8968-4899-8bf8-b796156e53d3, infoPort=40819, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=1725277916;c=1731993833712) 2024-11-19T05:23:57,083 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf1fb3ca2ed791a3c with lease ID 0xbf195cb27ed6189b: from storage DS-bdee27fb-08e5-40cc-b6ff-75a470d54fc7 node DatanodeRegistration(127.0.0.1:33169, datanodeUuid=101fec1c-8968-4899-8bf8-b796156e53d3, infoPort=40819, infoSecurePort=0, ipcPort=37781, storageInfo=lv=-57;cid=testClusterID;nsid=1725277916;c=1731993833712), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-19T05:23:57,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90 2024-11-19T05:23:57,185 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/zookeeper_0, clientPort=53542, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T05:23:57,194 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53542 2024-11-19T05:23:57,209 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:57,213 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:57,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741825_1001 (size=7) 2024-11-19T05:23:57,847 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad with version=8 2024-11-19T05:23:57,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/hbase-staging 2024-11-19T05:23:57,937 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T05:23:58,166 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f990f635ffd0:0 server-side Connection retries=45 2024-11-19T05:23:58,174 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,175 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T05:23:58,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T05:23:58,314 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T05:23:58,369 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T05:23:58,377 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T05:23:58,381 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T05:23:58,402 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 33061 (auto-detected) 2024-11-19T05:23:58,403 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T05:23:58,419 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34055 2024-11-19T05:23:58,438 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34055 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-19T05:23:58,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:340550x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T05:23:58,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34055-0x101518c9a1c0000 connected 2024-11-19T05:23:58,685 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:58,688 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:58,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T05:23:58,702 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad, hbase.cluster.distributed=false 2024-11-19T05:23:58,727 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T05:23:58,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34055 2024-11-19T05:23:58,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34055 2024-11-19T05:23:58,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34055 2024-11-19T05:23:58,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34055 2024-11-19T05:23:58,733 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34055 2024-11-19T05:23:58,829 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f990f635ffd0:0 server-side Connection retries=45 2024-11-19T05:23:58,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,831 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T05:23:58,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T05:23:58,831 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T05:23:58,834 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T05:23:58,836 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T05:23:58,837 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38449 2024-11-19T05:23:58,839 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38449 connecting to ZooKeeper ensemble=127.0.0.1:53542 2024-11-19T05:23:58,840 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:58,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:58,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384490x0, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T05:23:58,852 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384490x0, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T05:23:58,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38449-0x101518c9a1c0001 connected 2024-11-19T05:23:58,857 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T05:23:58,867 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T05:23:58,871 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T05:23:58,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T05:23:58,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38449 2024-11-19T05:23:58,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38449 2024-11-19T05:23:58,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38449 2024-11-19T05:23:58,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38449 2024-11-19T05:23:58,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38449 2024-11-19T05:23:58,894 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f990f635ffd0:34055 2024-11-19T05:23:58,895 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f990f635ffd0,34055,1731993838016 2024-11-19T05:23:58,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T05:23:58,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T05:23:58,906 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f990f635ffd0,34055,1731993838016 2024-11-19T05:23:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T05:23:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:23:58,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:23:58,936 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T05:23:58,937 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f990f635ffd0,34055,1731993838016 from backup master directory 2024-11-19T05:23:58,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T05:23:58,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f990f635ffd0,34055,1731993838016 2024-11-19T05:23:58,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T05:23:58,946 WARN [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T05:23:58,947 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f990f635ffd0,34055,1731993838016 2024-11-19T05:23:58,948 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T05:23:58,950 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T05:23:59,002 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/hbase.id] with ID: ec00e422-e444-4d95-8b94-b69e99cfec7d 2024-11-19T05:23:59,002 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/.tmp/hbase.id 2024-11-19T05:23:59,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741826_1002 (size=42) 2024-11-19T05:23:59,418 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/.tmp/hbase.id]:[hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/hbase.id] 2024-11-19T05:23:59,466 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-19T05:23:59,471 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T05:23:59,488 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-19T05:23:59,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:23:59,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:23:59,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741827_1003 (size=196) 2024-11-19T05:23:59,983 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:23:59,986 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T05:24:00,000 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:00,003 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T05:24:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741828_1004 (size=1189) 2024-11-19T05:24:00,449 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store 2024-11-19T05:24:00,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741829_1005 (size=34) 2024-11-19T05:24:00,471 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T05:24:00,474 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:00,475 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T05:24:00,475 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:24:00,475 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:24:00,477 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T05:24:00,478 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:24:00,478 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:24:00,479 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731993840475Disabling compacts and flushes for region at 1731993840475Disabling writes for close at 1731993840477 (+2 ms)Writing region close event to WAL at 1731993840478 (+1 ms)Closed at 1731993840478 2024-11-19T05:24:00,482 WARN [master/f990f635ffd0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/.initializing 2024-11-19T05:24:00,482 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/WALs/f990f635ffd0,34055,1731993838016 2024-11-19T05:24:00,490 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T05:24:00,503 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f990f635ffd0%2C34055%2C1731993838016, suffix=, logDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/WALs/f990f635ffd0,34055,1731993838016, archiveDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/oldWALs, maxLogs=10 2024-11-19T05:24:00,522 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/WALs/f990f635ffd0,34055,1731993838016/f990f635ffd0%2C34055%2C1731993838016.1731993840506, exclude list is [], retry=0 2024-11-19T05:24:00,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33169,DS-2c89d28b-24fb-4551-a99d-6fed4db29f11,DISK] 2024-11-19T05:24:00,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-19T05:24:00,573 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/WALs/f990f635ffd0,34055,1731993838016/f990f635ffd0%2C34055%2C1731993838016.1731993840506 2024-11-19T05:24:00,574 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-19T05:24:00,574 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:24:00,575 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:00,578 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,579 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T05:24:00,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:00,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:00,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T05:24:00,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:00,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:00,651 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T05:24:00,654 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:00,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:00,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T05:24:00,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:00,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:00,662 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,665 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,666 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,671 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,672 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,677 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T05:24:00,681 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T05:24:00,686 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:24:00,688 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60705892, jitterRate=-0.09541171789169312}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T05:24:00,695 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731993840590Initializing all the Stores at 1731993840592 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993840593 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993840593Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993840594 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993840594Cleaning up temporary data from old regions at 1731993840672 (+78 ms)Region opened successfully at 1731993840695 (+23 ms) 2024-11-19T05:24:00,696 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T05:24:00,738 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46394c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f990f635ffd0/172.17.0.2:0 2024-11-19T05:24:00,772 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T05:24:00,781 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T05:24:00,782 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T05:24:00,784 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T05:24:00,785 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T05:24:00,789 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-19T05:24:00,790 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T05:24:00,815 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T05:24:00,824 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T05:24:00,851 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T05:24:00,854 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T05:24:00,856 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T05:24:00,861 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T05:24:00,863 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T05:24:00,868 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T05:24:00,872 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T05:24:00,874 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T05:24:00,882 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T05:24:00,900 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T05:24:00,913 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T05:24:00,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T05:24:00,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T05:24:00,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:00,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:00,927 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f990f635ffd0,34055,1731993838016, sessionid=0x101518c9a1c0000, setting cluster-up flag (Was=false) 2024-11-19T05:24:00,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:00,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:00,988 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T05:24:00,990 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f990f635ffd0,34055,1731993838016 2024-11-19T05:24:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:01,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:01,146 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T05:24:01,148 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f990f635ffd0,34055,1731993838016 2024-11-19T05:24:01,155 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T05:24:01,187 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(746): ClusterId : ec00e422-e444-4d95-8b94-b69e99cfec7d 2024-11-19T05:24:01,189 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T05:24:01,201 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T05:24:01,201 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T05:24:01,210 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T05:24:01,211 DEBUG [RS:0;f990f635ffd0:38449 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@715c6771, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f990f635ffd0/172.17.0.2:0 2024-11-19T05:24:01,228 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f990f635ffd0:38449 2024-11-19T05:24:01,232 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T05:24:01,232 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T05:24:01,233 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T05:24:01,232 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T05:24:01,236 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(2659): reportForDuty to master=f990f635ffd0,34055,1731993838016 with port=38449, startcode=1731993838792 2024-11-19T05:24:01,245 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T05:24:01,250 DEBUG [RS:0;f990f635ffd0:38449 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T05:24:01,254 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T05:24:01,262 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f990f635ffd0,34055,1731993838016 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T05:24:01,279 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f990f635ffd0:0, corePoolSize=5, maxPoolSize=5 2024-11-19T05:24:01,279 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f990f635ffd0:0, corePoolSize=5, maxPoolSize=5 2024-11-19T05:24:01,280 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f990f635ffd0:0, corePoolSize=5, maxPoolSize=5 2024-11-19T05:24:01,280 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f990f635ffd0:0, corePoolSize=5, maxPoolSize=5 2024-11-19T05:24:01,280 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f990f635ffd0:0, corePoolSize=10, maxPoolSize=10 2024-11-19T05:24:01,280 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,281 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f990f635ffd0:0, corePoolSize=2, maxPoolSize=2 2024-11-19T05:24:01,281 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,297 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731993871297 2024-11-19T05:24:01,299 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56445, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T05:24:01,301 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T05:24:01,303 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T05:24:01,307 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34055 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:01,308 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T05:24:01,309 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T05:24:01,309 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T05:24:01,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T05:24:01,311 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T05:24:01,309 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T05:24:01,324 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:01,325 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T05:24:01,334 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,339 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T05:24:01,341 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T05:24:01,341 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T05:24:01,343 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-19T05:24:01,343 WARN [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-19T05:24:01,344 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T05:24:01,344 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T05:24:01,348 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.large.0-1731993841345,5,FailOnTimeoutGroup] 2024-11-19T05:24:01,352 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.small.0-1731993841348,5,FailOnTimeoutGroup] 2024-11-19T05:24:01,352 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,353 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T05:24:01,355 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741831_1007 (size=1321) 2024-11-19T05:24:01,356 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,359 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T05:24:01,360 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:24:01,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741832_1008 (size=32) 2024-11-19T05:24:01,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:01,392 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T05:24:01,396 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T05:24:01,396 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:01,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:01,398 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T05:24:01,402 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T05:24:01,402 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:01,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:01,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T05:24:01,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T05:24:01,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:01,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:01,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T05:24:01,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T05:24:01,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:01,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:01,415 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T05:24:01,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740 2024-11-19T05:24:01,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740 2024-11-19T05:24:01,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T05:24:01,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T05:24:01,422 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:24:01,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T05:24:01,437 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:24:01,441 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72053030, jitterRate=0.07367381453514099}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:24:01,444 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(2659): reportForDuty to master=f990f635ffd0,34055,1731993838016 with port=38449, startcode=1731993838792 2024-11-19T05:24:01,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731993841386Initializing all the Stores at 1731993841389 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993841389Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993841392 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993841392Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993841392Cleaning up temporary data from old regions at 1731993841421 (+29 ms)Region opened successfully at 1731993841446 (+25 ms) 2024-11-19T05:24:01,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34055 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T05:24:01,447 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T05:24:01,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T05:24:01,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T05:24:01,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T05:24:01,450 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34055 {}] master.ServerManager(517): Registering regionserver=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,451 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T05:24:01,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731993841447Disabling compacts and flushes for region at 1731993841447Disabling writes for close at 1731993841447Writing region close event to WAL at 1731993841451 (+4 ms)Closed at 1731993841451 2024-11-19T05:24:01,456 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T05:24:01,456 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T05:24:01,461 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:24:01,461 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45297 2024-11-19T05:24:01,461 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T05:24:01,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T05:24:01,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T05:24:01,473 DEBUG [RS:0;f990f635ffd0:38449 {}] zookeeper.ZKUtil(111): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,473 WARN [RS:0;f990f635ffd0:38449 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T05:24:01,473 INFO [RS:0;f990f635ffd0:38449 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T05:24:01,473 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T05:24:01,474 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,475 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f990f635ffd0,38449,1731993838792] 2024-11-19T05:24:01,477 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T05:24:01,505 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T05:24:01,525 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T05:24:01,532 INFO [RS:0;f990f635ffd0:38449 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T05:24:01,532 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,538 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T05:24:01,546 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T05:24:01,548 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,548 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,548 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f990f635ffd0:0, corePoolSize=2, maxPoolSize=2 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,549 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,550 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,550 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,550 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,550 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f990f635ffd0:0, corePoolSize=1, maxPoolSize=1 2024-11-19T05:24:01,550 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f990f635ffd0:0, corePoolSize=3, maxPoolSize=3 2024-11-19T05:24:01,551 DEBUG [RS:0;f990f635ffd0:38449 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0, corePoolSize=3, maxPoolSize=3 2024-11-19T05:24:01,552 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,552 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,552 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,553 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,553 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,553 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,38449,1731993838792-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T05:24:01,582 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T05:24:01,585 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,38449,1731993838792-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,586 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,586 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.Replication(171): f990f635ffd0,38449,1731993838792 started 2024-11-19T05:24:01,613 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:01,613 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1482): Serving as f990f635ffd0,38449,1731993838792, RpcServer on f990f635ffd0/172.17.0.2:38449, sessionid=0x101518c9a1c0001 2024-11-19T05:24:01,614 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T05:24:01,615 DEBUG [RS:0;f990f635ffd0:38449 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,615 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f990f635ffd0,38449,1731993838792' 2024-11-19T05:24:01,615 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T05:24:01,616 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T05:24:01,617 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T05:24:01,617 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T05:24:01,617 DEBUG [RS:0;f990f635ffd0:38449 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,617 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f990f635ffd0,38449,1731993838792' 2024-11-19T05:24:01,617 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T05:24:01,620 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T05:24:01,621 DEBUG [RS:0;f990f635ffd0:38449 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T05:24:01,622 INFO [RS:0;f990f635ffd0:38449 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T05:24:01,622 INFO [RS:0;f990f635ffd0:38449 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T05:24:01,628 WARN [f990f635ffd0:34055 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-19T05:24:01,726 INFO [RS:0;f990f635ffd0:38449 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T05:24:01,730 INFO [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f990f635ffd0%2C38449%2C1731993838792, suffix=, logDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792, archiveDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/oldWALs, maxLogs=32 2024-11-19T05:24:01,746 DEBUG [RS:0;f990f635ffd0:38449 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792/f990f635ffd0%2C38449%2C1731993838792.1731993841732, exclude list is [], retry=0 2024-11-19T05:24:01,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33169,DS-2c89d28b-24fb-4551-a99d-6fed4db29f11,DISK] 2024-11-19T05:24:01,756 INFO [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792/f990f635ffd0%2C38449%2C1731993838792.1731993841732 2024-11-19T05:24:01,757 DEBUG [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-19T05:24:01,881 DEBUG [f990f635ffd0:34055 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T05:24:01,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:01,906 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f990f635ffd0,38449,1731993838792, state=OPENING 2024-11-19T05:24:01,956 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T05:24:01,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:01,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:24:01,968 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T05:24:01,969 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T05:24:01,970 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T05:24:01,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:02,154 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T05:24:02,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47875, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T05:24:02,174 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T05:24:02,175 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T05:24:02,176 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T05:24:02,181 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f990f635ffd0%2C38449%2C1731993838792.meta, suffix=.meta, logDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792, archiveDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/oldWALs, maxLogs=32 2024-11-19T05:24:02,206 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792/f990f635ffd0%2C38449%2C1731993838792.meta.1731993842183.meta, exclude list is [], retry=0 2024-11-19T05:24:02,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33169,DS-2c89d28b-24fb-4551-a99d-6fed4db29f11,DISK] 2024-11-19T05:24:02,216 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/WALs/f990f635ffd0,38449,1731993838792/f990f635ffd0%2C38449%2C1731993838792.meta.1731993842183.meta 2024-11-19T05:24:02,217 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40819:40819)] 2024-11-19T05:24:02,217 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:24:02,220 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T05:24:02,224 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T05:24:02,230 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T05:24:02,235 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T05:24:02,236 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:02,237 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T05:24:02,237 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T05:24:02,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-19T05:24:02,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T05:24:02,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:02,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:02,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-19T05:24:02,247 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T05:24:02,248 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:02,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:02,249 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-19T05:24:02,252 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T05:24:02,252 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:02,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:02,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-19T05:24:02,256 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T05:24:02,256 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:02,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T05:24:02,257 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T05:24:02,260 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740 2024-11-19T05:24:02,264 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740 2024-11-19T05:24:02,268 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T05:24:02,268 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T05:24:02,269 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:24:02,278 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T05:24:02,282 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75108974, jitterRate=0.11921092867851257}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:24:02,282 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T05:24:02,284 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731993842238Writing region info on filesystem at 1731993842238Initializing all the Stores at 1731993842240 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993842241 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993842241Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993842241Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731993842241Cleaning up temporary data from old regions at 1731993842268 (+27 ms)Running coprocessor post-open hooks at 1731993842282 (+14 ms)Region opened successfully at 1731993842284 (+2 ms) 2024-11-19T05:24:02,292 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731993842143 2024-11-19T05:24:02,308 DEBUG [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T05:24:02,309 INFO [RS_OPEN_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T05:24:02,312 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:02,316 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f990f635ffd0,38449,1731993838792, state=OPEN 2024-11-19T05:24:02,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T05:24:02,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T05:24:02,353 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T05:24:02,354 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T05:24:02,354 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:02,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T05:24:02,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f990f635ffd0,38449,1731993838792 in 382 msec 2024-11-19T05:24:02,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T05:24:02,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 902 msec 2024-11-19T05:24:02,377 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T05:24:02,377 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T05:24:02,402 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:02,403 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:02,426 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:02,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52655, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:02,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2700 sec 2024-11-19T05:24:02,456 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731993842456, completionTime=-1 2024-11-19T05:24:02,460 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-19T05:24:02,461 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-19T05:24:02,537 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-19T05:24:02,538 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731993902538 2024-11-19T05:24:02,538 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731993962538 2024-11-19T05:24:02,538 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 77 msec 2024-11-19T05:24:02,541 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,542 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,542 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,544 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f990f635ffd0:34055, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,545 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,547 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,553 DEBUG [master/f990f635ffd0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T05:24:02,575 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.628sec 2024-11-19T05:24:02,576 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T05:24:02,578 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T05:24:02,579 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T05:24:02,579 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T05:24:02,579 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T05:24:02,580 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T05:24:02,581 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T05:24:02,590 DEBUG [master/f990f635ffd0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T05:24:02,591 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T05:24:02,591 INFO [master/f990f635ffd0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f990f635ffd0,34055,1731993838016-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T05:24:02,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:02,607 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T05:24:02,607 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T05:24:02,609 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:02,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:02,619 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40a2a69c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:02,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:02,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:02,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:02,629 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:02,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:02,632 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:02,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:02,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:02,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f990f635ffd0,34055,1731993838016 2024-11-19T05:24:02,677 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=405, ProcessCount=11, AvailableMemoryMB=13340 2024-11-19T05:24:02,685 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:24:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:24:02,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@23cc1f5f 2024-11-19T05:24:02,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:24:02,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:24:02,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:24:02,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:24:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:02,719 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:24:02,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-19T05:24:02,723 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:02,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:24:02,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:02,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741835_1011 (size=959) 2024-11-19T05:24:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:03,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:03,142 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:24:03,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741836_1012 (size=53) 2024-11-19T05:24:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:03,553 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:03,553 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 911bdd839d983af04003ffd0fc873824, disabling compactions & flushes 2024-11-19T05:24:03,553 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. after waiting 0 ms 2024-11-19T05:24:03,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,554 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,554 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 911bdd839d983af04003ffd0fc873824: Waiting for close lock at 1731993843553Disabling compacts and flushes for region at 1731993843553Disabling writes for close at 1731993843554 (+1 ms)Writing region close event to WAL at 1731993843554Closed at 1731993843554 2024-11-19T05:24:03,557 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:24:03,564 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993843557"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993843557"}]},"ts":"1731993843557"} 2024-11-19T05:24:03,572 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:24:03,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:24:03,580 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993843577"}]},"ts":"1731993843577"} 2024-11-19T05:24:03,586 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:24:03,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, ASSIGN}] 2024-11-19T05:24:03,592 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, ASSIGN 2024-11-19T05:24:03,595 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:24:03,746 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=911bdd839d983af04003ffd0fc873824, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:03,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, ASSIGN because future has completed 2024-11-19T05:24:03,752 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:03,920 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,920 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:24:03,920 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,921 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:03,921 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,921 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,924 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,926 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 911bdd839d983af04003ffd0fc873824 columnFamilyName A 2024-11-19T05:24:03,927 DEBUG [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:03,928 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(327): Store=911bdd839d983af04003ffd0fc873824/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:03,928 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,931 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 911bdd839d983af04003ffd0fc873824 columnFamilyName B 2024-11-19T05:24:03,931 DEBUG [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:03,932 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(327): Store=911bdd839d983af04003ffd0fc873824/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:03,932 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,935 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 911bdd839d983af04003ffd0fc873824 columnFamilyName C 2024-11-19T05:24:03,936 DEBUG [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:03,936 INFO [StoreOpener-911bdd839d983af04003ffd0fc873824-1 {}] regionserver.HStore(327): Store=911bdd839d983af04003ffd0fc873824/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:03,937 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,939 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,939 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,941 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,942 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,942 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:24:03,945 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,949 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:24:03,950 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 911bdd839d983af04003ffd0fc873824; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68969022, jitterRate=0.02771851420402527}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:24:03,950 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:03,951 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 911bdd839d983af04003ffd0fc873824: Running coprocessor pre-open hook at 1731993843921Writing region info on filesystem at 1731993843921Initializing all the Stores at 1731993843923 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993843923Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993843923Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993843923Cleaning up temporary data from old regions at 1731993843942 (+19 ms)Running coprocessor post-open hooks at 1731993843950 (+8 ms)Region opened successfully at 1731993843951 (+1 ms) 2024-11-19T05:24:03,953 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., pid=6, masterSystemTime=1731993843910 2024-11-19T05:24:03,957 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,957 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:03,958 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=911bdd839d983af04003ffd0fc873824, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:03,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:24:03,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T05:24:03,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 in 213 msec 2024-11-19T05:24:03,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T05:24:03,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, ASSIGN in 382 msec 2024-11-19T05:24:03,977 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:24:03,978 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993843977"}]},"ts":"1731993843977"} 2024-11-19T05:24:03,982 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:24:03,984 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:24:03,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2750 sec 2024-11-19T05:24:04,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T05:24:04,889 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:04,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f75e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcb8366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,906 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:04,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:04,915 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:24:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:24:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@545a7bf9 2024-11-19T05:24:04,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:24:04,923 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:24:04,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,930 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,933 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,933 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b8be13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,935 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,937 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,939 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0184e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,943 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cbf42c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,946 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59cb0f4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,955 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603c95bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b6b9e8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c9e84c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b8091e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,977 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,978 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783de3df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,982 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,983 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347812b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad2755a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,990 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1475e077, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:04,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:04,994 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:04,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be3f202, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:04,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:04,998 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:04,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@739049a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:05,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:05,007 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:05,007 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3e6f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:05,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:05,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:05,011 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:05,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194603ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:05,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:05,016 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:05,016 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5abee282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:05,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:05,018 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:05,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51ceab7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:05,030 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,030 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,040 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,042 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,053 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,057 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,060 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,075 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,078 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,086 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,092 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,109 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,109 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:05,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:05,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:05,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:05,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:24:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-19T05:24:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:05,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:05,137 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:05,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:05,170 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:05,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:05,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:05,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4a508b78b4f8426b8bb796e610edd41e is 50, key is test_row_0/A:col10/1731993845120/Put/seqid=0 2024-11-19T05:24:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741837_1013 (size=16681) 2024-11-19T05:24:05,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993905340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993905358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,378 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993905363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993905371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993905368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:05,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993905489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993905489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,506 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,507 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993905507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993905506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993905505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:05,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:05,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993905708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,715 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993905718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993905728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993905728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993905728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:05,752 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4a508b78b4f8426b8bb796e610edd41e 2024-11-19T05:24:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:05,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:05,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/2c3961b7d5134ab0a516138a25188d02 is 50, key is test_row_0/B:col10/1731993845120/Put/seqid=0 2024-11-19T05:24:05,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741838_1014 (size=12001) 2024-11-19T05:24:05,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/2c3961b7d5134ab0a516138a25188d02 2024-11-19T05:24:05,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b6c5d62f8bba427fb984b27d29cde018 is 50, key is test_row_0/C:col10/1731993845120/Put/seqid=0 2024-11-19T05:24:05,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:05,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:06,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741839_1015 (size=12001) 2024-11-19T05:24:06,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b6c5d62f8bba427fb984b27d29cde018 2024-11-19T05:24:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993906028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993906033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993906038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993906039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993906039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4a508b78b4f8426b8bb796e610edd41e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e 2024-11-19T05:24:06,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e, entries=250, sequenceid=13, filesize=16.3 K 2024-11-19T05:24:06,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/2c3961b7d5134ab0a516138a25188d02 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02 2024-11-19T05:24:06,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T05:24:06,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b6c5d62f8bba427fb984b27d29cde018 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018 2024-11-19T05:24:06,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018, entries=150, sequenceid=13, filesize=11.7 K 2024-11-19T05:24:06,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:06,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:06,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:06,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:06,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 911bdd839d983af04003ffd0fc873824 in 965ms, sequenceid=13, compaction requested=false 2024-11-19T05:24:06,144 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T05:24:06,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:06,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:06,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-19T05:24:06,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:06,297 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:24:06,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ea3b854d63114cee926532426361f981 is 50, key is test_row_0/A:col10/1731993845364/Put/seqid=0 2024-11-19T05:24:06,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741840_1016 (size=12001) 2024-11-19T05:24:06,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:06,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:06,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993906584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993906585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993906591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993906592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993906596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993906709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993906709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993906718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993906719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993906719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,778 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ea3b854d63114cee926532426361f981 2024-11-19T05:24:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/59e8f11f9b8545cabb1d476db05ab92a is 50, key is test_row_0/B:col10/1731993845364/Put/seqid=0 2024-11-19T05:24:06,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741841_1017 (size=12001) 2024-11-19T05:24:06,870 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/59e8f11f9b8545cabb1d476db05ab92a 2024-11-19T05:24:06,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/caf727f5ea7a428c98fc79275d6bd65d is 50, key is test_row_0/C:col10/1731993845364/Put/seqid=0 2024-11-19T05:24:06,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993906924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993906927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741842_1018 (size=12001) 2024-11-19T05:24:06,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993906939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993906938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:06,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:06,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993906940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:06,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993907239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993907239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993907248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993907258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993907259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:07,341 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/caf727f5ea7a428c98fc79275d6bd65d 2024-11-19T05:24:07,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ea3b854d63114cee926532426361f981 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981 2024-11-19T05:24:07,373 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:24:07,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/59e8f11f9b8545cabb1d476db05ab92a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a 2024-11-19T05:24:07,389 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:24:07,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/caf727f5ea7a428c98fc79275d6bd65d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d 2024-11-19T05:24:07,409 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:24:07,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 911bdd839d983af04003ffd0fc873824 in 1114ms, sequenceid=37, compaction requested=false 2024-11-19T05:24:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:07,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-19T05:24:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-19T05:24:07,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T05:24:07,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2760 sec 2024-11-19T05:24:07,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3040 sec 2024-11-19T05:24:07,749 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:24:07,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:07,771 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:07,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:07,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/6655b022248c43b686699f9157766aa1 is 50, key is test_row_0/A:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:07,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741843_1019 (size=12001) 2024-11-19T05:24:07,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/6655b022248c43b686699f9157766aa1 2024-11-19T05:24:07,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993907837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993907841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993907855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993907859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,871 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T05:24:07,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/451ff887c5fb4fdcabddd4d789eb8a18 is 50, key is test_row_0/B:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:07,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993907850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741844_1020 (size=12001) 2024-11-19T05:24:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993907968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993907968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993907978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993907978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:07,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993907988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:07,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993908178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993908178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993908188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993908188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993908198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/451ff887c5fb4fdcabddd4d789eb8a18 2024-11-19T05:24:08,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79e86f71b2904a46a2c464687de47a8f is 50, key is test_row_0/C:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:08,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741845_1021 (size=12001) 2024-11-19T05:24:08,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79e86f71b2904a46a2c464687de47a8f 2024-11-19T05:24:08,367 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T05:24:08,367 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T05:24:08,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/6655b022248c43b686699f9157766aa1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1 2024-11-19T05:24:08,371 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T05:24:08,371 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T05:24:08,372 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T05:24:08,372 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T05:24:08,374 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T05:24:08,374 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T05:24:08,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T05:24:08,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/451ff887c5fb4fdcabddd4d789eb8a18 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18 2024-11-19T05:24:08,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T05:24:08,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79e86f71b2904a46a2c464687de47a8f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f 2024-11-19T05:24:08,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-19T05:24:08,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 911bdd839d983af04003ffd0fc873824 in 651ms, sequenceid=50, compaction requested=true 2024-11-19T05:24:08,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:08,432 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:08,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:08,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:08,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:08,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:08,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:08,438 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:08,438 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:08,442 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:08,445 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:08,445 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:08,446 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.2 K 2024-11-19T05:24:08,448 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:08,448 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:08,448 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c3961b7d5134ab0a516138a25188d02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731993845085 2024-11-19T05:24:08,448 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:08,448 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=39.7 K 2024-11-19T05:24:08,449 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 59e8f11f9b8545cabb1d476db05ab92a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993845323 2024-11-19T05:24:08,450 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a508b78b4f8426b8bb796e610edd41e, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731993845085 2024-11-19T05:24:08,450 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 451ff887c5fb4fdcabddd4d789eb8a18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:08,451 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea3b854d63114cee926532426361f981, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993845323 2024-11-19T05:24:08,453 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6655b022248c43b686699f9157766aa1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:08,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:08,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:24:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:08,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/e011ae6cb7a34ef4a9789c06a86d730c is 50, key is test_row_0/A:col10/1731993848490/Put/seqid=0 2024-11-19T05:24:08,510 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#9 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:08,510 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:08,511 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/57080d506ef7418ba23046b42e9cf24f is 50, key is test_row_0/B:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:08,512 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/c1f13a7bf7e4452db027dbca64622111 is 50, key is test_row_0/A:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:08,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741846_1022 (size=12104) 2024-11-19T05:24:08,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993908532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993908535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993908532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,547 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741847_1023 (size=14341) 2024-11-19T05:24:08,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/e011ae6cb7a34ef4a9789c06a86d730c 2024-11-19T05:24:08,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993908542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,556 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/57080d506ef7418ba23046b42e9cf24f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/57080d506ef7418ba23046b42e9cf24f 2024-11-19T05:24:08,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993908545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741848_1024 (size=12104) 2024-11-19T05:24:08,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/573dcf269caa45139c29fb2b771fdc6c is 50, key is test_row_0/B:col10/1731993848490/Put/seqid=0 2024-11-19T05:24:08,589 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 57080d506ef7418ba23046b42e9cf24f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:08,589 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:08,595 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993848437; duration=0sec 2024-11-19T05:24:08,596 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:08,596 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:08,596 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:08,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741849_1025 (size=12001) 2024-11-19T05:24:08,605 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:08,606 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:08,607 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:08,607 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.2 K 2024-11-19T05:24:08,609 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c5d62f8bba427fb984b27d29cde018, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731993845085 2024-11-19T05:24:08,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/573dcf269caa45139c29fb2b771fdc6c 2024-11-19T05:24:08,611 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting caf727f5ea7a428c98fc79275d6bd65d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993845323 2024-11-19T05:24:08,617 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e86f71b2904a46a2c464687de47a8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:08,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c98af6c078eb42a1913f3bf0913509cd is 50, key is test_row_0/C:col10/1731993848490/Put/seqid=0 2024-11-19T05:24:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993908658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993908658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993908658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741850_1026 (size=12001) 2024-11-19T05:24:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993908667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993908668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,679 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c98af6c078eb42a1913f3bf0913509cd 2024-11-19T05:24:08,681 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/beadca3fe89246b58a627df1928b0dac is 50, key is test_row_0/C:col10/1731993847764/Put/seqid=0 2024-11-19T05:24:08,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/e011ae6cb7a34ef4a9789c06a86d730c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c 2024-11-19T05:24:08,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741851_1027 (size=12104) 2024-11-19T05:24:08,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c, entries=200, sequenceid=74, filesize=14.0 K 2024-11-19T05:24:08,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/573dcf269caa45139c29fb2b771fdc6c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c 2024-11-19T05:24:08,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-19T05:24:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c98af6c078eb42a1913f3bf0913509cd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd 2024-11-19T05:24:08,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd, entries=150, sequenceid=74, filesize=11.7 K 2024-11-19T05:24:08,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 911bdd839d983af04003ffd0fc873824 in 272ms, sequenceid=74, compaction requested=false 2024-11-19T05:24:08,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:08,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:08,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:24:08,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0b1ed8c71e314a30835ec9194e61974a is 50, key is test_row_0/A:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:08,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741852_1028 (size=16681) 2024-11-19T05:24:08,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0b1ed8c71e314a30835ec9194e61974a 2024-11-19T05:24:08,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7be0bbe2047c49e2bf608d8995f70f77 is 50, key is test_row_0/B:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993908927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993908937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993908938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993908945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741853_1029 (size=12001) 2024-11-19T05:24:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993908945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:08,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:08,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7be0bbe2047c49e2bf608d8995f70f77 2024-11-19T05:24:08,984 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/c1f13a7bf7e4452db027dbca64622111 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/c1f13a7bf7e4452db027dbca64622111 2024-11-19T05:24:08,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8b3202efd32041cfa1291b64bbe047f8 is 50, key is test_row_0/C:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:09,002 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into c1f13a7bf7e4452db027dbca64622111(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:09,002 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,002 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993848424; duration=0sec 2024-11-19T05:24:09,002 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:09,002 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:09,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741854_1030 (size=12001) 2024-11-19T05:24:09,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8b3202efd32041cfa1291b64bbe047f8 2024-11-19T05:24:09,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0b1ed8c71e314a30835ec9194e61974a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a 2024-11-19T05:24:09,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a, entries=250, sequenceid=89, filesize=16.3 K 2024-11-19T05:24:09,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7be0bbe2047c49e2bf608d8995f70f77 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77 2024-11-19T05:24:09,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T05:24:09,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993909057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993909058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993909058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993909057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993909058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8b3202efd32041cfa1291b64bbe047f8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8 2024-11-19T05:24:09,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8, entries=150, sequenceid=89, filesize=11.7 K 2024-11-19T05:24:09,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 911bdd839d983af04003ffd0fc873824 in 226ms, sequenceid=89, compaction requested=true 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:09,099 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:24:09,101 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:09,101 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:09,102 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,102 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/c1f13a7bf7e4452db027dbca64622111, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=42.1 K 2024-11-19T05:24:09,104 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f13a7bf7e4452db027dbca64622111, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:09,105 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e011ae6cb7a34ef4a9789c06a86d730c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731993847818 2024-11-19T05:24:09,105 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1ed8c71e314a30835ec9194e61974a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848526 2024-11-19T05:24:09,130 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#18 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:09,131 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5d328a5ba3574ea28e3188cd3358771f is 50, key is test_row_0/A:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:09,139 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/beadca3fe89246b58a627df1928b0dac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/beadca3fe89246b58a627df1928b0dac 2024-11-19T05:24:09,163 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into beadca3fe89246b58a627df1928b0dac(size=11.8 K), total size for store is 35.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:09,164 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,164 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993848437; duration=0sec 2024-11-19T05:24:09,164 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:24:09,164 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:09,164 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:09,164 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:09,172 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:09,172 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:09,172 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,174 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/57080d506ef7418ba23046b42e9cf24f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.3 K 2024-11-19T05:24:09,176 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 57080d506ef7418ba23046b42e9cf24f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:09,178 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 573dcf269caa45139c29fb2b771fdc6c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731993847818 2024-11-19T05:24:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741855_1031 (size=12207) 2024-11-19T05:24:09,180 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7be0bbe2047c49e2bf608d8995f70f77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848541 2024-11-19T05:24:09,205 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:09,206 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/8ae12b8d84724acbbe856ad941cef9b3 is 50, key is test_row_0/B:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:09,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741856_1032 (size=12207) 2024-11-19T05:24:09,253 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/8ae12b8d84724acbbe856ad941cef9b3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8ae12b8d84724acbbe856ad941cef9b3 2024-11-19T05:24:09,278 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 8ae12b8d84724acbbe856ad941cef9b3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:09,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,280 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993849099; duration=0sec 2024-11-19T05:24:09,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:09,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:09,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:09,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:09,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:24:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:09,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:09,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:09,286 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,286 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/beadca3fe89246b58a627df1928b0dac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.3 K 2024-11-19T05:24:09,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-19T05:24:09,288 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:09,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:09,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/d54645942bb243a0afe3bac69765e0d7 is 50, key is test_row_0/A:col10/1731993848920/Put/seqid=0 2024-11-19T05:24:09,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-19T05:24:09,294 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting beadca3fe89246b58a627df1928b0dac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731993846578 2024-11-19T05:24:09,297 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:09,298 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c98af6c078eb42a1913f3bf0913509cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731993847818 2024-11-19T05:24:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T05:24:09,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:09,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:09,301 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b3202efd32041cfa1291b64bbe047f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848541 2024-11-19T05:24:09,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741857_1033 (size=14341) 2024-11-19T05:24:09,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993909318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993909322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/d54645942bb243a0afe3bac69765e0d7 2024-11-19T05:24:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993909326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,340 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,341 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/43c565056552414188834c43d102fd0a is 50, key is test_row_0/C:col10/1731993848868/Put/seqid=0 2024-11-19T05:24:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993909327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993909326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/88447e3c9d154a33a6bf55190b7fbf9a is 50, key is test_row_0/B:col10/1731993848920/Put/seqid=0 2024-11-19T05:24:09,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741858_1034 (size=12207) 2024-11-19T05:24:09,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741859_1035 (size=12001) 2024-11-19T05:24:09,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T05:24:09,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/88447e3c9d154a33a6bf55190b7fbf9a 2024-11-19T05:24:09,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/d7f322474dcb4d16b4e84a88e493330f is 50, key is test_row_0/C:col10/1731993848920/Put/seqid=0 2024-11-19T05:24:09,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993909438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993909438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993909465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993909465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993909466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T05:24:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:09,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741860_1036 (size=12001) 2024-11-19T05:24:09,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/d7f322474dcb4d16b4e84a88e493330f 2024-11-19T05:24:09,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/d54645942bb243a0afe3bac69765e0d7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7 2024-11-19T05:24:09,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7, entries=200, sequenceid=115, filesize=14.0 K 2024-11-19T05:24:09,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/88447e3c9d154a33a6bf55190b7fbf9a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a 2024-11-19T05:24:09,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T05:24:09,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/d7f322474dcb4d16b4e84a88e493330f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f 2024-11-19T05:24:09,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f, entries=150, sequenceid=115, filesize=11.7 K 2024-11-19T05:24:09,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 911bdd839d983af04003ffd0fc873824 in 297ms, sequenceid=115, compaction requested=false 2024-11-19T05:24:09,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,593 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5d328a5ba3574ea28e3188cd3358771f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5d328a5ba3574ea28e3188cd3358771f 2024-11-19T05:24:09,607 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into 5d328a5ba3574ea28e3188cd3358771f(size=11.9 K), total size for store is 25.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:09,607 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,607 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993849099; duration=0sec 2024-11-19T05:24:09,607 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:09,607 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T05:24:09,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-19T05:24:09,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,632 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:24:09,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/fb292aa3e7a342acb5914ed83bdd7e59 is 50, key is test_row_0/A:col10/1731993849318/Put/seqid=0 2024-11-19T05:24:09,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:09,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741861_1037 (size=12001) 2024-11-19T05:24:09,682 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/fb292aa3e7a342acb5914ed83bdd7e59 2024-11-19T05:24:09,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/411681c6544f4cd9a4f1608433b441ac is 50, key is test_row_0/B:col10/1731993849318/Put/seqid=0 2024-11-19T05:24:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993909735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993909735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993909739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,750 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993909750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993909752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741862_1038 (size=12001) 2024-11-19T05:24:09,768 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/411681c6544f4cd9a4f1608433b441ac 2024-11-19T05:24:09,791 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/43c565056552414188834c43d102fd0a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/43c565056552414188834c43d102fd0a 2024-11-19T05:24:09,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/6cc55fa4b729427f95925ce99978f7e6 is 50, key is test_row_0/C:col10/1731993849318/Put/seqid=0 2024-11-19T05:24:09,806 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into 43c565056552414188834c43d102fd0a(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:09,806 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,807 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993849099; duration=0sec 2024-11-19T05:24:09,807 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:09,807 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741863_1039 (size=12001) 2024-11-19T05:24:09,844 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/6cc55fa4b729427f95925ce99978f7e6 2024-11-19T05:24:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/fb292aa3e7a342acb5914ed83bdd7e59 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59 2024-11-19T05:24:09,868 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59, entries=150, sequenceid=130, filesize=11.7 K 2024-11-19T05:24:09,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993909858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/411681c6544f4cd9a4f1608433b441ac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac 2024-11-19T05:24:09,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993909858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993909858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993909859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993909869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:09,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:09,889 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac, entries=150, sequenceid=130, filesize=11.7 K 2024-11-19T05:24:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/6cc55fa4b729427f95925ce99978f7e6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6 2024-11-19T05:24:09,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6, entries=150, sequenceid=130, filesize=11.7 K 2024-11-19T05:24:09,916 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 911bdd839d983af04003ffd0fc873824 in 284ms, sequenceid=130, compaction requested=true 2024-11-19T05:24:09,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:09,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:09,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-19T05:24:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-19T05:24:09,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-19T05:24:09,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 620 msec 2024-11-19T05:24:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T05:24:09,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 636 msec 2024-11-19T05:24:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:10,089 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:10,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:24:10,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f5ecb2b51ac44a2b967d07ec2301684b is 50, key is test_row_0/A:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:10,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993910116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993910120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741864_1040 (size=16931) 2024-11-19T05:24:10,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f5ecb2b51ac44a2b967d07ec2301684b 2024-11-19T05:24:10,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993910135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993910137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993910137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7cd12eea96cd4b97aa818df1fd9cde04 is 50, key is test_row_0/B:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:10,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741865_1041 (size=12151) 2024-11-19T05:24:10,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7cd12eea96cd4b97aa818df1fd9cde04 2024-11-19T05:24:10,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/1d9bfb8b5e94411b922bb41574ca4e8c is 50, key is test_row_0/C:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:10,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993910249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993910258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993910258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993910258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993910270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741866_1042 (size=12151) 2024-11-19T05:24:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-19T05:24:10,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:10,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-19T05:24:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:10,445 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:10,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:10,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:10,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993910457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993910478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993910478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993910478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993910478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:10,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T05:24:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:10,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:10,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:10,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:10,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/1d9bfb8b5e94411b922bb41574ca4e8c 2024-11-19T05:24:10,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f5ecb2b51ac44a2b967d07ec2301684b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b 2024-11-19T05:24:10,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b, entries=250, sequenceid=156, filesize=16.5 K 2024-11-19T05:24:10,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7cd12eea96cd4b97aa818df1fd9cde04 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04 2024-11-19T05:24:10,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:24:10,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/1d9bfb8b5e94411b922bb41574ca4e8c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c 2024-11-19T05:24:10,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:24:10,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 911bdd839d983af04003ffd0fc873824 in 661ms, sequenceid=156, compaction requested=true 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:10,751 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:10,751 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:10,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:10,753 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:10,753 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:10,754 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:10,754 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:10,754 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:10,754 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:10,754 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5d328a5ba3574ea28e3188cd3358771f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=54.2 K 2024-11-19T05:24:10,754 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8ae12b8d84724acbbe856ad941cef9b3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=47.2 K 2024-11-19T05:24:10,755 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d328a5ba3574ea28e3188cd3358771f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848541 2024-11-19T05:24:10,755 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ae12b8d84724acbbe856ad941cef9b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848541 2024-11-19T05:24:10,756 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 88447e3c9d154a33a6bf55190b7fbf9a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731993848920 2024-11-19T05:24:10,756 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting d54645942bb243a0afe3bac69765e0d7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731993848913 2024-11-19T05:24:10,756 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb292aa3e7a342acb5914ed83bdd7e59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731993849318 2024-11-19T05:24:10,757 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 411681c6544f4cd9a4f1608433b441ac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731993849318 2024-11-19T05:24:10,757 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5ecb2b51ac44a2b967d07ec2301684b, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:10,758 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cd12eea96cd4b97aa818df1fd9cde04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:10,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-19T05:24:10,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:10,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:10,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0df961a848fb4284ba82d49ca0ff2d0a is 50, key is test_row_0/A:col10/1731993850130/Put/seqid=0 2024-11-19T05:24:10,779 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:10,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/b03a2e20a92f4e3eac110fee90b18709 is 50, key is test_row_0/B:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:10,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:10,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:10,816 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:10,818 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4e9d11444e3446039f910c64264431db is 50, key is test_row_0/A:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:10,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741867_1043 (size=12151) 2024-11-19T05:24:10,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741868_1044 (size=12493) 2024-11-19T05:24:10,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741869_1045 (size=12493) 2024-11-19T05:24:10,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993910838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993910843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993910915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993910912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993910917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:10,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993911017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993911028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993911037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993911038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993911037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:11,233 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0df961a848fb4284ba82d49ca0ff2d0a 2024-11-19T05:24:11,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993911228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/98a78c09d03042b99d6c223815221850 is 50, key is test_row_0/B:col10/1731993850130/Put/seqid=0 2024-11-19T05:24:11,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993911258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993911247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993911258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,272 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/b03a2e20a92f4e3eac110fee90b18709 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/b03a2e20a92f4e3eac110fee90b18709 2024-11-19T05:24:11,278 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4e9d11444e3446039f910c64264431db as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4e9d11444e3446039f910c64264431db 2024-11-19T05:24:11,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993911259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,285 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into b03a2e20a92f4e3eac110fee90b18709(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:11,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:11,285 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=12, startTime=1731993850751; duration=0sec 2024-11-19T05:24:11,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:11,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:11,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:11,288 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:11,288 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:11,288 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:11,289 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/43c565056552414188834c43d102fd0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=47.2 K 2024-11-19T05:24:11,289 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 43c565056552414188834c43d102fd0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731993848541 2024-11-19T05:24:11,290 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d7f322474dcb4d16b4e84a88e493330f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731993848920 2024-11-19T05:24:11,291 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cc55fa4b729427f95925ce99978f7e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731993849318 2024-11-19T05:24:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,292 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into 4e9d11444e3446039f910c64264431db(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:11,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,293 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d9bfb8b5e94411b922bb41574ca4e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:11,293 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:11,293 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=12, startTime=1731993850751; duration=0sec 2024-11-19T05:24:11,294 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:11,294 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:11,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741870_1046 (size=12151) 2024-11-19T05:24:11,311 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/98a78c09d03042b99d6c223815221850 2024-11-19T05:24:11,331 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#34 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:11,331 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b7256f82e6fc4b0790b9926c6b6b03fd is 50, key is test_row_0/C:col10/1731993850080/Put/seqid=0 2024-11-19T05:24:11,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b41b71a71f0a40ec88e133a345fcaae6 is 50, key is test_row_0/C:col10/1731993850130/Put/seqid=0 2024-11-19T05:24:11,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741871_1047 (size=12493) 2024-11-19T05:24:11,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741872_1048 (size=12151) 2024-11-19T05:24:11,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993911548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993911567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993911567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993911577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:11,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993911598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:11,801 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b7256f82e6fc4b0790b9926c6b6b03fd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b7256f82e6fc4b0790b9926c6b6b03fd 2024-11-19T05:24:11,803 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b41b71a71f0a40ec88e133a345fcaae6 2024-11-19T05:24:11,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/0df961a848fb4284ba82d49ca0ff2d0a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a 2024-11-19T05:24:11,825 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into b7256f82e6fc4b0790b9926c6b6b03fd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:11,825 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:11,825 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=12, startTime=1731993850751; duration=0sec 2024-11-19T05:24:11,825 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:11,825 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:11,829 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T05:24:11,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/98a78c09d03042b99d6c223815221850 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850 2024-11-19T05:24:11,841 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T05:24:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/b41b71a71f0a40ec88e133a345fcaae6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6 2024-11-19T05:24:11,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6, entries=150, sequenceid=169, filesize=11.9 K 2024-11-19T05:24:11,857 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 911bdd839d983af04003ffd0fc873824 in 1099ms, sequenceid=169, compaction requested=false 2024-11-19T05:24:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:11,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-19T05:24:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-19T05:24:11,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-19T05:24:11,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-19T05:24:11,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4280 sec 2024-11-19T05:24:12,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:12,075 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:12,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T05:24:12,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8cb503849e1c4d76a9d663149dab73f0 is 50, key is test_row_0/A:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993912100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993912103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993912108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993912112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993912118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741873_1049 (size=14541) 2024-11-19T05:24:12,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993912218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993912227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993912227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993912227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,231 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993912427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993912438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993912438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993912439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8cb503849e1c4d76a9d663149dab73f0 2024-11-19T05:24:12,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e0e131a2e1b34824bf20b6c73d8f718f is 50, key is test_row_0/B:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-19T05:24:12,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:12,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:12,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-19T05:24:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:12,593 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:12,594 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:12,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:12,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741874_1050 (size=12151) 2024-11-19T05:24:12,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e0e131a2e1b34824bf20b6c73d8f718f 2024-11-19T05:24:12,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/18a760748b054bbf9a5ed2eb246bf5bb is 50, key is test_row_0/C:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741875_1051 (size=12151) 2024-11-19T05:24:12,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/18a760748b054bbf9a5ed2eb246bf5bb 2024-11-19T05:24:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8cb503849e1c4d76a9d663149dab73f0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0 2024-11-19T05:24:12,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0, entries=200, sequenceid=198, filesize=14.2 K 2024-11-19T05:24:12,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e0e131a2e1b34824bf20b6c73d8f718f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f 2024-11-19T05:24:12,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T05:24:12,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/18a760748b054bbf9a5ed2eb246bf5bb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb 2024-11-19T05:24:12,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb, entries=150, sequenceid=198, filesize=11.9 K 2024-11-19T05:24:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:12,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 911bdd839d983af04003ffd0fc873824 in 625ms, sequenceid=198, compaction requested=true 2024-11-19T05:24:12,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:12,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:12,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:12,700 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:12,700 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:12,702 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:12,702 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:12,702 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,703 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4e9d11444e3446039f910c64264431db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=38.3 K 2024-11-19T05:24:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:12,703 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:12,703 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:12,704 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,704 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/b03a2e20a92f4e3eac110fee90b18709, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.9 K 2024-11-19T05:24:12,704 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e9d11444e3446039f910c64264431db, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:12,704 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b03a2e20a92f4e3eac110fee90b18709, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:12,705 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a78c09d03042b99d6c223815221850, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731993850112 2024-11-19T05:24:12,705 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0df961a848fb4284ba82d49ca0ff2d0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731993850112 2024-11-19T05:24:12,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:12,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:12,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:12,706 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e131a2e1b34824bf20b6c73d8f718f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:12,707 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cb503849e1c4d76a9d663149dab73f0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:12,738 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#39 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:12,739 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/311243f1ec0e4e3db6b350cdb38070e0 is 50, key is test_row_0/A:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:12,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:12,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,752 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#40 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:12,753 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/9aeff4426c6944afbf4e542a1a006f29 is 50, key is test_row_0/B:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/73ab3fdb36e745229981da55c5829c12 is 50, key is test_row_0/A:col10/1731993852107/Put/seqid=0 2024-11-19T05:24:12,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741876_1052 (size=12595) 2024-11-19T05:24:12,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741877_1053 (size=12595) 2024-11-19T05:24:12,824 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/311243f1ec0e4e3db6b350cdb38070e0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/311243f1ec0e4e3db6b350cdb38070e0 2024-11-19T05:24:12,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993912820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993912820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993912825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993912829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,838 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/9aeff4426c6944afbf4e542a1a006f29 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/9aeff4426c6944afbf4e542a1a006f29 2024-11-19T05:24:12,838 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into 311243f1ec0e4e3db6b350cdb38070e0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:12,839 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:12,839 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993852700; duration=0sec 2024-11-19T05:24:12,840 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:12,840 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:12,841 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:12,845 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:12,845 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:12,845 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,845 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b7256f82e6fc4b0790b9926c6b6b03fd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=35.9 K 2024-11-19T05:24:12,852 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 9aeff4426c6944afbf4e542a1a006f29(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:12,852 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:12,852 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993852700; duration=0sec 2024-11-19T05:24:12,852 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:12,852 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:12,855 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7256f82e6fc4b0790b9926c6b6b03fd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993849720 2024-11-19T05:24:12,856 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b41b71a71f0a40ec88e133a345fcaae6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731993850112 2024-11-19T05:24:12,856 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a760748b054bbf9a5ed2eb246bf5bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:12,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741878_1054 (size=12151) 2024-11-19T05:24:12,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/73ab3fdb36e745229981da55c5829c12 2024-11-19T05:24:12,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/850af3ddd40a4b49ba91f7b8f45e4346 is 50, key is test_row_0/B:col10/1731993852107/Put/seqid=0 2024-11-19T05:24:12,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741879_1055 (size=12151) 2024-11-19T05:24:12,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/850af3ddd40a4b49ba91f7b8f45e4346 2024-11-19T05:24:12,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:12,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:12,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:12,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:12,912 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#43 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:12,913 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/879b8ca96b984dcabd603beff2514813 is 50, key is test_row_0/C:col10/1731993850842/Put/seqid=0 2024-11-19T05:24:12,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/92fad9b980a54299884c2215b60e38c2 is 50, key is test_row_0/C:col10/1731993852107/Put/seqid=0 2024-11-19T05:24:12,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993912938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993912939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993912938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993912948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:12,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:12,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741880_1056 (size=12595) 2024-11-19T05:24:12,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741881_1057 (size=12151) 2024-11-19T05:24:12,973 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/879b8ca96b984dcabd603beff2514813 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/879b8ca96b984dcabd603beff2514813 2024-11-19T05:24:12,986 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into 879b8ca96b984dcabd603beff2514813(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:12,986 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:12,986 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993852705; duration=0sec 2024-11-19T05:24:12,987 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:12,987 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:13,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:13,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993913137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,144 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993913148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993913147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993913148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993913157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:13,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/92fad9b980a54299884c2215b60e38c2 2024-11-19T05:24:13,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/73ab3fdb36e745229981da55c5829c12 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12 2024-11-19T05:24:13,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:13,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12, entries=150, sequenceid=209, filesize=11.9 K 2024-11-19T05:24:13,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/850af3ddd40a4b49ba91f7b8f45e4346 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346 2024-11-19T05:24:13,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346, entries=150, sequenceid=209, filesize=11.9 K 2024-11-19T05:24:13,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/92fad9b980a54299884c2215b60e38c2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2 2024-11-19T05:24:13,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2, entries=150, sequenceid=209, filesize=11.9 K 2024-11-19T05:24:13,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 911bdd839d983af04003ffd0fc873824 in 655ms, sequenceid=209, compaction requested=false 2024-11-19T05:24:13,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:13,463 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:13,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T05:24:13,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/a674a0f1a7374123a834d5ffdba91a60 is 50, key is test_row_0/A:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993913475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741882_1058 (size=12151) 2024-11-19T05:24:13,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993913477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/a674a0f1a7374123a834d5ffdba91a60 2024-11-19T05:24:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993913478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993913478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/05674e37d53e4c90967ee9cba2a92fda is 50, key is test_row_0/B:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741883_1059 (size=12151) 2024-11-19T05:24:13,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/05674e37d53e4c90967ee9cba2a92fda 2024-11-19T05:24:13,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:13,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/38268a009fee4688a6ce5f096a5bdd6a is 50, key is test_row_0/C:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741884_1060 (size=12151) 2024-11-19T05:24:13,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/38268a009fee4688a6ce5f096a5bdd6a 2024-11-19T05:24:13,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/a674a0f1a7374123a834d5ffdba91a60 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60 2024-11-19T05:24:13,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60, entries=150, sequenceid=240, filesize=11.9 K 2024-11-19T05:24:13,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/05674e37d53e4c90967ee9cba2a92fda as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda 2024-11-19T05:24:13,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda, entries=150, sequenceid=240, filesize=11.9 K 2024-11-19T05:24:13,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/38268a009fee4688a6ce5f096a5bdd6a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a 2024-11-19T05:24:13,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993913588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993913588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993913588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993913588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a, entries=150, sequenceid=240, filesize=11.9 K 2024-11-19T05:24:13,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=46.96 KB/48090 for 911bdd839d983af04003ffd0fc873824 in 140ms, sequenceid=240, compaction requested=true 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:13,604 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:13,604 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:13,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:13,605 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:13,605 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:13,605 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:13,605 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:13,606 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,606 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,606 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/9aeff4426c6944afbf4e542a1a006f29, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.0 K 2024-11-19T05:24:13,606 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/311243f1ec0e4e3db6b350cdb38070e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.0 K 2024-11-19T05:24:13,606 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aeff4426c6944afbf4e542a1a006f29, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:13,606 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 311243f1ec0e4e3db6b350cdb38070e0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:13,607 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73ab3fdb36e745229981da55c5829c12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731993852099 2024-11-19T05:24:13,608 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a674a0f1a7374123a834d5ffdba91a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:13,608 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 850af3ddd40a4b49ba91f7b8f45e4346, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731993852099 2024-11-19T05:24:13,610 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 05674e37d53e4c90967ee9cba2a92fda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:13,636 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#48 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:13,637 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/e5d3de2664134093b604650cd061d318 is 50, key is test_row_0/A:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,658 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:13,659 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/1977aa5479484ad79e5da70b93139fc1 is 50, key is test_row_0/B:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741885_1061 (size=12697) 2024-11-19T05:24:13,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-19T05:24:13,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,678 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-19T05:24:13,689 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/e5d3de2664134093b604650cd061d318 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e5d3de2664134093b604650cd061d318 2024-11-19T05:24:13,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/933c5e7cfafb452c9d06d2c3e6bbf069 is 50, key is test_row_0/A:col10/1731993853475/Put/seqid=0 2024-11-19T05:24:13,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741886_1062 (size=12697) 2024-11-19T05:24:13,698 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into e5d3de2664134093b604650cd061d318(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:13,698 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:13,698 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993853604; duration=0sec 2024-11-19T05:24:13,698 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:13,698 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:13,698 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:13,700 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:13,700 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:13,700 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:13,701 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/879b8ca96b984dcabd603beff2514813, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.0 K 2024-11-19T05:24:13,701 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879b8ca96b984dcabd603beff2514813, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731993850842 2024-11-19T05:24:13,702 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92fad9b980a54299884c2215b60e38c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731993852099 2024-11-19T05:24:13,702 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38268a009fee4688a6ce5f096a5bdd6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:13,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741887_1063 (size=12151) 2024-11-19T05:24:13,715 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#51 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:13,715 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/ab79a71d2c934317b8b9adba07b1575d is 50, key is test_row_0/C:col10/1731993852826/Put/seqid=0 2024-11-19T05:24:13,724 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/933c5e7cfafb452c9d06d2c3e6bbf069 2024-11-19T05:24:13,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741888_1064 (size=12697) 2024-11-19T05:24:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:13,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/3097a1d03e5d4153a6aa57eea7038c01 is 50, key is test_row_0/B:col10/1731993853475/Put/seqid=0 2024-11-19T05:24:13,743 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/ab79a71d2c934317b8b9adba07b1575d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/ab79a71d2c934317b8b9adba07b1575d 2024-11-19T05:24:13,753 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into ab79a71d2c934317b8b9adba07b1575d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:13,753 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:13,753 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993853604; duration=0sec 2024-11-19T05:24:13,753 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:13,753 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:13,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741889_1065 (size=12151) 2024-11-19T05:24:13,756 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/3097a1d03e5d4153a6aa57eea7038c01 2024-11-19T05:24:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/31f4be6378674a0dbe7f28a2450ec9db is 50, key is test_row_0/C:col10/1731993853475/Put/seqid=0 2024-11-19T05:24:13,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741890_1066 (size=12151) 2024-11-19T05:24:13,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:13,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993913846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993913854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993913854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993913858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993913957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993913968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993913968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:13,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:13,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993913968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:13,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,102 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/1977aa5479484ad79e5da70b93139fc1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1977aa5479484ad79e5da70b93139fc1 2024-11-19T05:24:14,110 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 1977aa5479484ad79e5da70b93139fc1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:14,110 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:14,110 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993853604; duration=0sec 2024-11-19T05:24:14,111 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:14,111 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:14,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993914167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993914177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993914177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993914177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,184 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/31f4be6378674a0dbe7f28a2450ec9db 2024-11-19T05:24:14,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/933c5e7cfafb452c9d06d2c3e6bbf069 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069 2024-11-19T05:24:14,198 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T05:24:14,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/3097a1d03e5d4153a6aa57eea7038c01 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01 2024-11-19T05:24:14,210 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T05:24:14,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/31f4be6378674a0dbe7f28a2450ec9db as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db 2024-11-19T05:24:14,219 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db, entries=150, sequenceid=250, filesize=11.9 K 2024-11-19T05:24:14,222 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=167.72 KB/171750 for 911bdd839d983af04003ffd0fc873824 in 543ms, sequenceid=250, compaction requested=false 2024-11-19T05:24:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:14,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-19T05:24:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-19T05:24:14,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-19T05:24:14,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6300 sec 2024-11-19T05:24:14,236 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6430 sec 2024-11-19T05:24:14,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:14,482 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:14,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-19T05:24:14,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ccf6b6561af8481ebb2e3c6f35aa5852 is 50, key is test_row_0/A:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:14,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993914488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993914491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993914494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993914494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741891_1067 (size=14741) 2024-11-19T05:24:14,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ccf6b6561af8481ebb2e3c6f35aa5852 2024-11-19T05:24:14,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7e6603fcd8ae4d41b9b080ecbf2361ae is 50, key is test_row_0/B:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:14,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741892_1068 (size=12301) 2024-11-19T05:24:14,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993914597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993914608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993914608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-19T05:24:14,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:14,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-19T05:24:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:14,739 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:14,740 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:14,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:14,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993914807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993914817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993914818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:14,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:14,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:14,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:14,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7e6603fcd8ae4d41b9b080ecbf2361ae 2024-11-19T05:24:14,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/cf1003aaf90a4ff0994efc1601bd8892 is 50, key is test_row_0/C:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:14,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741893_1069 (size=12301) 2024-11-19T05:24:14,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/cf1003aaf90a4ff0994efc1601bd8892 2024-11-19T05:24:14,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/ccf6b6561af8481ebb2e3c6f35aa5852 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852 2024-11-19T05:24:14,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852, entries=200, sequenceid=282, filesize=14.4 K 2024-11-19T05:24:14,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/7e6603fcd8ae4d41b9b080ecbf2361ae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae 2024-11-19T05:24:14,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T05:24:14,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/cf1003aaf90a4ff0994efc1601bd8892 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892 2024-11-19T05:24:14,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892, entries=150, sequenceid=282, filesize=12.0 K 2024-11-19T05:24:14,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 911bdd839d983af04003ffd0fc873824 in 494ms, sequenceid=282, compaction requested=true 2024-11-19T05:24:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:14,976 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:14,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:14,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:14,977 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:14,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:14,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:14,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:14,978 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:14,978 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:14,978 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:14,978 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:14,978 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:14,978 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:14,978 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e5d3de2664134093b604650cd061d318, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=38.7 K 2024-11-19T05:24:14,978 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1977aa5479484ad79e5da70b93139fc1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.3 K 2024-11-19T05:24:14,979 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5d3de2664134093b604650cd061d318, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:14,979 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1977aa5479484ad79e5da70b93139fc1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:14,979 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933c5e7cfafb452c9d06d2c3e6bbf069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731993853468 2024-11-19T05:24:14,979 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 3097a1d03e5d4153a6aa57eea7038c01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731993853468 2024-11-19T05:24:14,979 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccf6b6561af8481ebb2e3c6f35aa5852, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:14,980 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e6603fcd8ae4d41b9b080ecbf2361ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:14,991 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:14,992 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f02d618e95994f9197653a1411b60199 is 50, key is test_row_0/A:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:14,994 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:14,994 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/0614001428a7468b8433d4477a207a4a is 50, key is test_row_0/B:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:15,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741895_1071 (size=12949) 2024-11-19T05:24:15,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741894_1070 (size=12949) 2024-11-19T05:24:15,028 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/0614001428a7468b8433d4477a207a4a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0614001428a7468b8433d4477a207a4a 2024-11-19T05:24:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:15,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:15,038 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 0614001428a7468b8433d4477a207a4a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:15,038 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:15,038 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993854976; duration=0sec 2024-11-19T05:24:15,038 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:15,038 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:15,038 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:15,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/7ba0be9721754078975f266909c3c2b4 is 50, key is test_row_0/A:col10/1731993855029/Put/seqid=0 2024-11-19T05:24:15,042 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:15,042 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:15,042 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,042 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/ab79a71d2c934317b8b9adba07b1575d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.3 K 2024-11-19T05:24:15,043 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ab79a71d2c934317b8b9adba07b1575d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731993852826 2024-11-19T05:24:15,044 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f4be6378674a0dbe7f28a2450ec9db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731993853468 2024-11-19T05:24:15,044 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1003aaf90a4ff0994efc1601bd8892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:15,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741896_1072 (size=12301) 2024-11-19T05:24:15,069 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:15,070 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/dc643afc86704ed68a701f77b04f201b is 50, key is test_row_0/C:col10/1731993853850/Put/seqid=0 2024-11-19T05:24:15,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741897_1073 (size=12949) 2024-11-19T05:24:15,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993915132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993915134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993915136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993915137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,141 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993915157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,162 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993915247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993915247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993915247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993915247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:15,435 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f02d618e95994f9197653a1411b60199 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f02d618e95994f9197653a1411b60199 2024-11-19T05:24:15,442 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into f02d618e95994f9197653a1411b60199(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:15,442 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:15,442 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993854976; duration=0sec 2024-11-19T05:24:15,444 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:15,444 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:15,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993915457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993915457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,458 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993915457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993915458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/7ba0be9721754078975f266909c3c2b4 2024-11-19T05:24:15,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f680689136c44885a94d1e8d3072e4b9 is 50, key is test_row_0/B:col10/1731993855029/Put/seqid=0 2024-11-19T05:24:15,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741898_1074 (size=12301) 2024-11-19T05:24:15,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f680689136c44885a94d1e8d3072e4b9 2024-11-19T05:24:15,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79bc4753f61146d7b9362bbf5359d072 is 50, key is test_row_0/C:col10/1731993855029/Put/seqid=0 2024-11-19T05:24:15,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741899_1075 (size=12301) 2024-11-19T05:24:15,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,521 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/dc643afc86704ed68a701f77b04f201b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dc643afc86704ed68a701f77b04f201b 2024-11-19T05:24:15,527 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into dc643afc86704ed68a701f77b04f201b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:15,527 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:15,527 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993854977; duration=0sec 2024-11-19T05:24:15,528 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:15,528 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:15,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993915767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993915767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993915767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:15,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993915768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:15,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:15,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:15,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79bc4753f61146d7b9362bbf5359d072 2024-11-19T05:24:15,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/7ba0be9721754078975f266909c3c2b4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4 2024-11-19T05:24:15,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T05:24:15,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f680689136c44885a94d1e8d3072e4b9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9 2024-11-19T05:24:15,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T05:24:15,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/79bc4753f61146d7b9362bbf5359d072 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072 2024-11-19T05:24:15,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072, entries=150, sequenceid=293, filesize=12.0 K 2024-11-19T05:24:15,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 911bdd839d983af04003ffd0fc873824 in 895ms, sequenceid=293, compaction requested=false 2024-11-19T05:24:15,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:15,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-19T05:24:15,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:15,977 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:24:15,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/29bb2035aae34ffd8bae9c25eca79667 is 50, key is test_row_0/A:col10/1731993855121/Put/seqid=0 2024-11-19T05:24:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741900_1076 (size=12301) 2024-11-19T05:24:15,994 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/29bb2035aae34ffd8bae9c25eca79667 2024-11-19T05:24:16,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e9d4c849b4774b6989eb347fee22ed27 is 50, key is test_row_0/B:col10/1731993855121/Put/seqid=0 2024-11-19T05:24:16,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741901_1077 (size=12301) 2024-11-19T05:24:16,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:16,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993916287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993916288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993916292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993916292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993916397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993916407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993916407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,412 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e9d4c849b4774b6989eb347fee22ed27 2024-11-19T05:24:16,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c373031459284e2d887e4163fa82f530 is 50, key is test_row_0/C:col10/1731993855121/Put/seqid=0 2024-11-19T05:24:16,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741902_1078 (size=12301) 2024-11-19T05:24:16,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993916607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993916617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993916617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,828 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c373031459284e2d887e4163fa82f530 2024-11-19T05:24:16,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/29bb2035aae34ffd8bae9c25eca79667 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667 2024-11-19T05:24:16,840 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T05:24:16,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e9d4c849b4774b6989eb347fee22ed27 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27 2024-11-19T05:24:16,847 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T05:24:16,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/c373031459284e2d887e4163fa82f530 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530 2024-11-19T05:24:16,858 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530, entries=150, sequenceid=321, filesize=12.0 K 2024-11-19T05:24:16,859 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 911bdd839d983af04003ffd0fc873824 in 881ms, sequenceid=321, compaction requested=true 2024-11-19T05:24:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-19T05:24:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-19T05:24:16,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-19T05:24:16,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1210 sec 2024-11-19T05:24:16,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1270 sec 2024-11-19T05:24:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-19T05:24:16,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:16,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:16,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-19T05:24:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:16,892 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:16,894 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:16,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:16,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:16,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:16,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:16,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8aea1dbd1e4740ce9ddf41d6690cedca is 50, key is test_row_0/A:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741903_1079 (size=14741) 2024-11-19T05:24:16,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8aea1dbd1e4740ce9ddf41d6690cedca 2024-11-19T05:24:16,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e7eae5c231304927aec00de7341345de is 50, key is test_row_0/B:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:16,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741904_1080 (size=12301) 2024-11-19T05:24:16,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e7eae5c231304927aec00de7341345de 2024-11-19T05:24:16,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/283a2f2f18c04dd3a0af978a5a56d484 is 50, key is test_row_0/C:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:16,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993916974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993916974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:16,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993916975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:16,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:16,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741905_1081 (size=12301) 2024-11-19T05:24:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:17,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T05:24:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993917088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993917088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993917088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T05:24:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:17,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993917297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993917298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993917298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993917304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,305 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T05:24:17,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:17,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/283a2f2f18c04dd3a0af978a5a56d484 2024-11-19T05:24:17,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/8aea1dbd1e4740ce9ddf41d6690cedca as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca 2024-11-19T05:24:17,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca, entries=200, sequenceid=334, filesize=14.4 K 2024-11-19T05:24:17,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e7eae5c231304927aec00de7341345de as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de 2024-11-19T05:24:17,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T05:24:17,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/283a2f2f18c04dd3a0af978a5a56d484 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484 2024-11-19T05:24:17,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484, entries=150, sequenceid=334, filesize=12.0 K 2024-11-19T05:24:17,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 911bdd839d983af04003ffd0fc873824 in 529ms, sequenceid=334, compaction requested=true 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:17,449 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:17,449 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:17,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:17,452 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:17,452 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:17,452 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,452 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0614001428a7468b8433d4477a207a4a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=48.7 K 2024-11-19T05:24:17,453 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:17,453 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0614001428a7468b8433d4477a207a4a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:17,453 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:17,453 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,453 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f02d618e95994f9197653a1411b60199, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=51.1 K 2024-11-19T05:24:17,455 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f680689136c44885a94d1e8d3072e4b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731993854491 2024-11-19T05:24:17,455 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d4c849b4774b6989eb347fee22ed27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731993855077 2024-11-19T05:24:17,455 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f02d618e95994f9197653a1411b60199, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:17,456 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e7eae5c231304927aec00de7341345de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:17,456 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ba0be9721754078975f266909c3c2b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731993854491 2024-11-19T05:24:17,456 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29bb2035aae34ffd8bae9c25eca79667, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731993855077 2024-11-19T05:24:17,456 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aea1dbd1e4740ce9ddf41d6690cedca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:17,474 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#69 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:17,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e9106d2875c7445685df6e88e923b685 is 50, key is test_row_0/B:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:17,486 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:17,487 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/41b16f0efa304463b96a417c800a66c9 is 50, key is test_row_0/A:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:17,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741906_1082 (size=13085) 2024-11-19T05:24:17,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-19T05:24:17,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:17,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,517 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:24:17,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741907_1083 (size=13085) 2024-11-19T05:24:17,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4ffb9602d8d042daab89f22720f57480 is 50, key is test_row_0/A:col10/1731993856969/Put/seqid=0 2024-11-19T05:24:17,527 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e9106d2875c7445685df6e88e923b685 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9106d2875c7445685df6e88e923b685 2024-11-19T05:24:17,539 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/41b16f0efa304463b96a417c800a66c9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/41b16f0efa304463b96a417c800a66c9 2024-11-19T05:24:17,541 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into e9106d2875c7445685df6e88e923b685(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:17,541 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:17,541 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=12, startTime=1731993857449; duration=0sec 2024-11-19T05:24:17,541 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:17,541 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:17,542 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:17,545 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:17,545 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:17,545 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:17,545 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dc643afc86704ed68a701f77b04f201b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=48.7 K 2024-11-19T05:24:17,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting dc643afc86704ed68a701f77b04f201b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731993853838 2024-11-19T05:24:17,547 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into 41b16f0efa304463b96a417c800a66c9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:17,547 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:17,548 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=12, startTime=1731993857449; duration=0sec 2024-11-19T05:24:17,548 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:17,548 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:17,549 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 79bc4753f61146d7b9362bbf5359d072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731993854491 2024-11-19T05:24:17,550 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c373031459284e2d887e4163fa82f530, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731993855077 2024-11-19T05:24:17,551 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 283a2f2f18c04dd3a0af978a5a56d484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:17,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741908_1084 (size=12301) 2024-11-19T05:24:17,574 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4ffb9602d8d042daab89f22720f57480 2024-11-19T05:24:17,591 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:17,591 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/d5757f9077644ba38c9d840cddd466e8 is 50, key is test_row_0/C:col10/1731993856285/Put/seqid=0 2024-11-19T05:24:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c9df3beeee03499492746d85eb7f13ba is 50, key is test_row_0/B:col10/1731993856969/Put/seqid=0 2024-11-19T05:24:17,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741910_1086 (size=12301) 2024-11-19T05:24:17,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741909_1085 (size=13085) 2024-11-19T05:24:17,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:17,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:17,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993917628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993917630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993917630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993917737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993917737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993917747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993917947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993917947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:17,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993917957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:17,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,006 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c9df3beeee03499492746d85eb7f13ba 2024-11-19T05:24:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/aead6a2003a8408188add36332152ca8 is 50, key is test_row_0/C:col10/1731993856969/Put/seqid=0 2024-11-19T05:24:18,018 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/d5757f9077644ba38c9d840cddd466e8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d5757f9077644ba38c9d840cddd466e8 2024-11-19T05:24:18,026 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into d5757f9077644ba38c9d840cddd466e8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:18,026 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:18,026 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=12, startTime=1731993857449; duration=0sec 2024-11-19T05:24:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:18,027 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:18,027 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:18,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741911_1087 (size=12301) 2024-11-19T05:24:18,029 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/aead6a2003a8408188add36332152ca8 2024-11-19T05:24:18,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/4ffb9602d8d042daab89f22720f57480 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480 2024-11-19T05:24:18,045 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480, entries=150, sequenceid=357, filesize=12.0 K 2024-11-19T05:24:18,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c9df3beeee03499492746d85eb7f13ba as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba 2024-11-19T05:24:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba, entries=150, sequenceid=357, filesize=12.0 K 2024-11-19T05:24:18,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/aead6a2003a8408188add36332152ca8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8 2024-11-19T05:24:18,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8, entries=150, sequenceid=357, filesize=12.0 K 2024-11-19T05:24:18,063 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 911bdd839d983af04003ffd0fc873824 in 546ms, sequenceid=357, compaction requested=false 2024-11-19T05:24:18,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:18,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:18,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-19T05:24:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-19T05:24:18,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-19T05:24:18,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1710 sec 2024-11-19T05:24:18,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.1820 sec 2024-11-19T05:24:18,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:18,265 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:18,265 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:24:18,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9244082c9dbe48b2b257407f6033b3bb is 50, key is test_row_0/A:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741912_1088 (size=17181) 2024-11-19T05:24:18,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9244082c9dbe48b2b257407f6033b3bb 2024-11-19T05:24:18,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c7795e4ab853492a83eadffc9057ef1b is 50, key is test_row_0/B:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993918313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993918315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,320 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993918318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741913_1089 (size=12301) 2024-11-19T05:24:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993918428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993918429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993918430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993918638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993918638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:18,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993918643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:18,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c7795e4ab853492a83eadffc9057ef1b 2024-11-19T05:24:18,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/2cfbd1f29a004c7ba55de12baad0fb41 is 50, key is test_row_0/C:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741914_1090 (size=12301) 2024-11-19T05:24:18,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/2cfbd1f29a004c7ba55de12baad0fb41 2024-11-19T05:24:18,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9244082c9dbe48b2b257407f6033b3bb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb 2024-11-19T05:24:18,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb, entries=250, sequenceid=375, filesize=16.8 K 2024-11-19T05:24:18,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/c7795e4ab853492a83eadffc9057ef1b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b 2024-11-19T05:24:18,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-19T05:24:18,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/2cfbd1f29a004c7ba55de12baad0fb41 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41 2024-11-19T05:24:18,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41, entries=150, sequenceid=375, filesize=12.0 K 2024-11-19T05:24:18,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 911bdd839d983af04003ffd0fc873824 in 504ms, sequenceid=375, compaction requested=true 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:18,769 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:18,769 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:18,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:18,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:18,770 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42567 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:18,770 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:18,770 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:18,770 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:18,770 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:18,770 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:18,771 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/41b16f0efa304463b96a417c800a66c9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=41.6 K 2024-11-19T05:24:18,771 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9106d2875c7445685df6e88e923b685, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.8 K 2024-11-19T05:24:18,771 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41b16f0efa304463b96a417c800a66c9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:18,771 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e9106d2875c7445685df6e88e923b685, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:18,771 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ffb9602d8d042daab89f22720f57480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731993856969 2024-11-19T05:24:18,772 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c9df3beeee03499492746d85eb7f13ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731993856969 2024-11-19T05:24:18,772 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9244082c9dbe48b2b257407f6033b3bb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:18,772 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c7795e4ab853492a83eadffc9057ef1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:18,784 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:18,785 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/d2f54e506eb1402b8dddc466479f4e5b is 50, key is test_row_0/B:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,786 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:18,787 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/95837926ae66404593ffaf35719abbe3 is 50, key is test_row_0/A:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741916_1092 (size=13187) 2024-11-19T05:24:18,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741915_1091 (size=13187) 2024-11-19T05:24:18,805 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/d2f54e506eb1402b8dddc466479f4e5b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/d2f54e506eb1402b8dddc466479f4e5b 2024-11-19T05:24:18,812 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into d2f54e506eb1402b8dddc466479f4e5b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:18,812 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:18,812 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993858769; duration=0sec 2024-11-19T05:24:18,812 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:18,813 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:18,813 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:18,814 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:18,815 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:18,815 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:18,815 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d5757f9077644ba38c9d840cddd466e8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=36.8 K 2024-11-19T05:24:18,815 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d5757f9077644ba38c9d840cddd466e8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731993856285 2024-11-19T05:24:18,816 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting aead6a2003a8408188add36332152ca8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731993856969 2024-11-19T05:24:18,816 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cfbd1f29a004c7ba55de12baad0fb41, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:18,824 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#80 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:18,824 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/acd7e6a45b85407dad916d96daef62ad is 50, key is test_row_0/C:col10/1731993857622/Put/seqid=0 2024-11-19T05:24:18,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741917_1093 (size=13187) 2024-11-19T05:24:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:18,951 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:18,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:24:18,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/64d1cde35b3a4d3893b29cc45af2a2f5 is 50, key is test_row_0/A:col10/1731993858950/Put/seqid=0 2024-11-19T05:24:18,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741918_1094 (size=17181) 2024-11-19T05:24:19,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993919015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993919016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993919016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-19T05:24:19,038 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:19,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-19T05:24:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:19,040 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:19,041 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:19,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993919127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993919127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993919127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:19,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54970 deadline: 1731993919187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,190 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,203 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/95837926ae66404593ffaf35719abbe3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/95837926ae66404593ffaf35719abbe3 2024-11-19T05:24:19,211 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into 95837926ae66404593ffaf35719abbe3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:19,211 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:19,211 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993858769; duration=0sec 2024-11-19T05:24:19,211 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:19,211 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:19,237 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/acd7e6a45b85407dad916d96daef62ad as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/acd7e6a45b85407dad916d96daef62ad 2024-11-19T05:24:19,243 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into acd7e6a45b85407dad916d96daef62ad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:19,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:19,243 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993858769; duration=0sec 2024-11-19T05:24:19,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:19,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:19,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993919327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,329 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993919337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993919337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993919338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:19,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/64d1cde35b3a4d3893b29cc45af2a2f5 2024-11-19T05:24:19,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/1d83f3c071434eaa920dab04c9fb9ffb is 50, key is test_row_0/B:col10/1731993858950/Put/seqid=0 2024-11-19T05:24:19,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741919_1095 (size=12301) 2024-11-19T05:24:19,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993919647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993919647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993919647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:19,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:19,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:19,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/1d83f3c071434eaa920dab04c9fb9ffb 2024-11-19T05:24:19,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/9ce4fd2379464b29af18aa93b53e09c9 is 50, key is test_row_0/C:col10/1731993858950/Put/seqid=0 2024-11-19T05:24:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741920_1096 (size=12301) 2024-11-19T05:24:19,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:19,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:20,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:20,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:20,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:20,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:20,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:20,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:20,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993920157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993920157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:20,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993920158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:20,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:20,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/9ce4fd2379464b29af18aa93b53e09c9 2024-11-19T05:24:20,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/64d1cde35b3a4d3893b29cc45af2a2f5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5 2024-11-19T05:24:20,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5, entries=250, sequenceid=399, filesize=16.8 K 2024-11-19T05:24:20,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/1d83f3c071434eaa920dab04c9fb9ffb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb 2024-11-19T05:24:20,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb, entries=150, sequenceid=399, filesize=12.0 K 2024-11-19T05:24:20,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/9ce4fd2379464b29af18aa93b53e09c9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9 2024-11-19T05:24:20,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9, entries=150, sequenceid=399, filesize=12.0 K 2024-11-19T05:24:20,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 911bdd839d983af04003ffd0fc873824 in 1272ms, sequenceid=399, compaction requested=false 2024-11-19T05:24:20,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:20,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-19T05:24:20,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:20,269 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:24:20,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5213db4ad30b4ac497fa86fa22c1ca93 is 50, key is test_row_0/A:col10/1731993858963/Put/seqid=0 2024-11-19T05:24:20,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741921_1097 (size=12301) 2024-11-19T05:24:20,680 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5213db4ad30b4ac497fa86fa22c1ca93 2024-11-19T05:24:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f9c05fa55e56489aa85117a35e31f1a9 is 50, key is test_row_0/B:col10/1731993858963/Put/seqid=0 2024-11-19T05:24:20,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741922_1098 (size=12301) 2024-11-19T05:24:21,100 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f9c05fa55e56489aa85117a35e31f1a9 2024-11-19T05:24:21,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8292532b58444d858694754e5ee5315b is 50, key is test_row_0/C:col10/1731993858963/Put/seqid=0 2024-11-19T05:24:21,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741923_1099 (size=12301) 2024-11-19T05:24:21,117 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8292532b58444d858694754e5ee5315b 2024-11-19T05:24:21,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5213db4ad30b4ac497fa86fa22c1ca93 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93 2024-11-19T05:24:21,129 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T05:24:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/f9c05fa55e56489aa85117a35e31f1a9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9 2024-11-19T05:24:21,136 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T05:24:21,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/8292532b58444d858694754e5ee5315b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b 2024-11-19T05:24:21,143 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b, entries=150, sequenceid=416, filesize=12.0 K 2024-11-19T05:24:21,144 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 911bdd839d983af04003ffd0fc873824 in 875ms, sequenceid=416, compaction requested=true 2024-11-19T05:24:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-19T05:24:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-19T05:24:21,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-19T05:24:21,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1050 sec 2024-11-19T05:24:21,154 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.1140 sec 2024-11-19T05:24:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:21,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:21,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:21,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/79e3daee3a2a4277a7057d04a35805cb is 50, key is test_row_0/A:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-19T05:24:21,189 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:21,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:21,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-19T05:24:21,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:21,194 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:21,195 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:21,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:21,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741924_1100 (size=17181) 2024-11-19T05:24:21,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993921235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993921236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993921236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:21,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993921347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993921347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993921347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:21,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:21,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993921557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993921558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993921567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/79e3daee3a2a4277a7057d04a35805cb 2024-11-19T05:24:21,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/32cfca5498f345d4863e14075ffbc595 is 50, key is test_row_0/B:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:21,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741925_1101 (size=12301) 2024-11-19T05:24:21,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993921867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,872 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993921868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993921877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:21,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:21,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:22,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/32cfca5498f345d4863e14075ffbc595 2024-11-19T05:24:22,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/fd9e8b91ac2d49dcbdc31f410e032184 is 50, key is test_row_0/C:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:22,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741926_1102 (size=12301) 2024-11-19T05:24:22,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/fd9e8b91ac2d49dcbdc31f410e032184 2024-11-19T05:24:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/79e3daee3a2a4277a7057d04a35805cb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb 2024-11-19T05:24:22,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb, entries=250, sequenceid=427, filesize=16.8 K 2024-11-19T05:24:22,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/32cfca5498f345d4863e14075ffbc595 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595 2024-11-19T05:24:22,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T05:24:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/fd9e8b91ac2d49dcbdc31f410e032184 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184 2024-11-19T05:24:22,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184, entries=150, sequenceid=427, filesize=12.0 K 2024-11-19T05:24:22,078 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 911bdd839d983af04003ffd0fc873824 in 897ms, sequenceid=427, compaction requested=true 2024-11-19T05:24:22,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:22,078 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:22,079 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:22,080 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 59850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:22,080 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:22,080 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:22,080 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/95837926ae66404593ffaf35719abbe3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=58.4 K 2024-11-19T05:24:22,081 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95837926ae66404593ffaf35719abbe3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:22,082 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64d1cde35b3a4d3893b29cc45af2a2f5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731993858305 2024-11-19T05:24:22,082 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:22,082 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5213db4ad30b4ac497fa86fa22c1ca93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731993858963 2024-11-19T05:24:22,082 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:22,082 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:22,082 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/d2f54e506eb1402b8dddc466479f4e5b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=48.9 K 2024-11-19T05:24:22,082 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79e3daee3a2a4277a7057d04a35805cb, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861170 2024-11-19T05:24:22,083 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d2f54e506eb1402b8dddc466479f4e5b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:22,083 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d83f3c071434eaa920dab04c9fb9ffb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731993858305 2024-11-19T05:24:22,083 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f9c05fa55e56489aa85117a35e31f1a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731993858963 2024-11-19T05:24:22,084 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 32cfca5498f345d4863e14075ffbc595, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861175 2024-11-19T05:24:22,094 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:22,094 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f0d9796b52124d479c7c2fdd68987f2c is 50, key is test_row_0/A:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:22,096 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:22,096 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/329c76685d8046498a6e2223939dc939 is 50, key is test_row_0/B:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:22,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741928_1104 (size=13323) 2024-11-19T05:24:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741927_1103 (size=13323) 2024-11-19T05:24:22,112 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/329c76685d8046498a6e2223939dc939 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/329c76685d8046498a6e2223939dc939 2024-11-19T05:24:22,121 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/f0d9796b52124d479c7c2fdd68987f2c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f0d9796b52124d479c7c2fdd68987f2c 2024-11-19T05:24:22,125 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 329c76685d8046498a6e2223939dc939(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:22,125 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:22,125 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=12, startTime=1731993862079; duration=0sec 2024-11-19T05:24:22,125 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:22,125 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:22,126 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:22,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-19T05:24:22,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:22,126 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:24:22,129 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:22,129 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:22,130 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:22,130 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/acd7e6a45b85407dad916d96daef62ad, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=48.9 K 2024-11-19T05:24:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/71257bef8cae4687afacceda197e6598 is 50, key is test_row_0/A:col10/1731993861233/Put/seqid=0 2024-11-19T05:24:22,134 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into f0d9796b52124d479c7c2fdd68987f2c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:22,134 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:22,134 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=12, startTime=1731993862078; duration=0sec 2024-11-19T05:24:22,134 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:22,134 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:22,134 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting acd7e6a45b85407dad916d96daef62ad, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731993857622 2024-11-19T05:24:22,135 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ce4fd2379464b29af18aa93b53e09c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731993858305 2024-11-19T05:24:22,135 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8292532b58444d858694754e5ee5315b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731993858963 2024-11-19T05:24:22,136 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fd9e8b91ac2d49dcbdc31f410e032184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861175 2024-11-19T05:24:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741929_1105 (size=12301) 2024-11-19T05:24:22,160 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#93 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:22,160 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/0bdce35ab33f4c39b013eac98c4b76cb is 50, key is test_row_0/C:col10/1731993861178/Put/seqid=0 2024-11-19T05:24:22,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741930_1106 (size=13323) 2024-11-19T05:24:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:22,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. as already flushing 2024-11-19T05:24:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993922408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993922409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993922409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993922517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993922517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,519 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993922518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,544 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/71257bef8cae4687afacceda197e6598 2024-11-19T05:24:22,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e635c25577c54ddcab6cf8cbbe048cf1 is 50, key is test_row_0/B:col10/1731993861233/Put/seqid=0 2024-11-19T05:24:22,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741931_1107 (size=12301) 2024-11-19T05:24:22,573 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/0bdce35ab33f4c39b013eac98c4b76cb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/0bdce35ab33f4c39b013eac98c4b76cb 2024-11-19T05:24:22,580 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into 0bdce35ab33f4c39b013eac98c4b76cb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:22,580 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:22,580 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=12, startTime=1731993862079; duration=0sec 2024-11-19T05:24:22,580 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:22,580 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993922727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993922727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993922727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:22,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:22,967 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e635c25577c54ddcab6cf8cbbe048cf1 2024-11-19T05:24:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/510baa451f9e408ea0b5ac4c6e454781 is 50, key is test_row_0/C:col10/1731993861233/Put/seqid=0 2024-11-19T05:24:22,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741932_1108 (size=12301) 2024-11-19T05:24:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993923037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993923037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993923037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54916 deadline: 1731993923337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,341 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 911bdd839d983af04003ffd0fc873824 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:23,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/510baa451f9e408ea0b5ac4c6e454781 2024-11-19T05:24:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/71257bef8cae4687afacceda197e6598 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598 2024-11-19T05:24:23,391 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598, entries=150, sequenceid=453, filesize=12.0 K 2024-11-19T05:24:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/e635c25577c54ddcab6cf8cbbe048cf1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1 2024-11-19T05:24:23,396 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1, entries=150, sequenceid=453, filesize=12.0 K 2024-11-19T05:24:23,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/510baa451f9e408ea0b5ac4c6e454781 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781 2024-11-19T05:24:23,403 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781, entries=150, sequenceid=453, filesize=12.0 K 2024-11-19T05:24:23,404 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 911bdd839d983af04003ffd0fc873824 in 1278ms, sequenceid=453, compaction requested=false 2024-11-19T05:24:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:23,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-19T05:24:23,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-19T05:24:23,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-19T05:24:23,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2140 sec 2024-11-19T05:24:23,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2250 sec 2024-11-19T05:24:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:23,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:23,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9bd977716d0249dba26f82480032612d is 50, key is test_row_0/A:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:23,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741933_1109 (size=14741) 2024-11-19T05:24:23,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993923581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,585 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993923584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993923585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993923697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993923697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,699 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993923697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993923907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993923907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,909 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993923907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:23,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:23,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9bd977716d0249dba26f82480032612d 2024-11-19T05:24:23,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/0584b1904d6948adbeebe0df1ea6ad9c is 50, key is test_row_0/B:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:23,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741934_1110 (size=12301) 2024-11-19T05:24:24,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993924218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993924218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993924218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/0584b1904d6948adbeebe0df1ea6ad9c 2024-11-19T05:24:24,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/7ddd943dfe2b452da4e42f5703db67cc is 50, key is test_row_0/C:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:24,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741935_1111 (size=12301) 2024-11-19T05:24:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54948 deadline: 1731993924728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55000 deadline: 1731993924728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54932 deadline: 1731993924729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:24,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:24:24,792 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/7ddd943dfe2b452da4e42f5703db67cc 2024-11-19T05:24:24,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/9bd977716d0249dba26f82480032612d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d 2024-11-19T05:24:24,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d, entries=200, sequenceid=469, filesize=14.4 K 2024-11-19T05:24:24,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/0584b1904d6948adbeebe0df1ea6ad9c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c 2024-11-19T05:24:24,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c, entries=150, sequenceid=469, filesize=12.0 K 2024-11-19T05:24:24,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/7ddd943dfe2b452da4e42f5703db67cc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc 2024-11-19T05:24:24,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc, entries=150, sequenceid=469, filesize=12.0 K 2024-11-19T05:24:24,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 911bdd839d983af04003ffd0fc873824 in 1281ms, sequenceid=469, compaction requested=true 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:24,829 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:24,829 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 911bdd839d983af04003ffd0fc873824:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:24,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:24,831 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:24,831 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:24,831 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/A is initiating minor compaction (all files) 2024-11-19T05:24:24,831 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/B is initiating minor compaction (all files) 2024-11-19T05:24:24,831 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/B in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:24,831 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/329c76685d8046498a6e2223939dc939, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=37.0 K 2024-11-19T05:24:24,831 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/A in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:24,831 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f0d9796b52124d479c7c2fdd68987f2c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=39.4 K 2024-11-19T05:24:24,832 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 329c76685d8046498a6e2223939dc939, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861175 2024-11-19T05:24:24,832 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0d9796b52124d479c7c2fdd68987f2c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861175 2024-11-19T05:24:24,832 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e635c25577c54ddcab6cf8cbbe048cf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731993861215 2024-11-19T05:24:24,833 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71257bef8cae4687afacceda197e6598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731993861215 2024-11-19T05:24:24,833 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0584b1904d6948adbeebe0df1ea6ad9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731993862397 2024-11-19T05:24:24,834 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd977716d0249dba26f82480032612d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731993862397 2024-11-19T05:24:24,845 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:24,845 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/a8704bab26644f2aa4a3fcb6cb895ca0 is 50, key is test_row_0/A:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:24,849 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:24,850 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/4026f01e25fd455fb3e11893f78f69a3 is 50, key is test_row_0/B:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:24,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741936_1112 (size=13425) 2024-11-19T05:24:24,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741937_1113 (size=13425) 2024-11-19T05:24:24,863 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/a8704bab26644f2aa4a3fcb6cb895ca0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a8704bab26644f2aa4a3fcb6cb895ca0 2024-11-19T05:24:24,868 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/4026f01e25fd455fb3e11893f78f69a3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/4026f01e25fd455fb3e11893f78f69a3 2024-11-19T05:24:24,875 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/A of 911bdd839d983af04003ffd0fc873824 into a8704bab26644f2aa4a3fcb6cb895ca0(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:24,875 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:24,875 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/A, priority=13, startTime=1731993864829; duration=0sec 2024-11-19T05:24:24,875 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:24,875 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:A 2024-11-19T05:24:24,875 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:24,877 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:24,877 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 911bdd839d983af04003ffd0fc873824/C is initiating minor compaction (all files) 2024-11-19T05:24:24,878 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 911bdd839d983af04003ffd0fc873824/C in TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:24,878 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/0bdce35ab33f4c39b013eac98c4b76cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp, totalSize=37.0 K 2024-11-19T05:24:24,879 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/B of 911bdd839d983af04003ffd0fc873824 into 4026f01e25fd455fb3e11893f78f69a3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:24,879 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:24,879 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/B, priority=13, startTime=1731993864829; duration=0sec 2024-11-19T05:24:24,879 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdce35ab33f4c39b013eac98c4b76cb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1731993861175 2024-11-19T05:24:24,879 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:24,879 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:B 2024-11-19T05:24:24,883 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 510baa451f9e408ea0b5ac4c6e454781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731993861215 2024-11-19T05:24:24,884 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ddd943dfe2b452da4e42f5703db67cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731993862397 2024-11-19T05:24:24,899 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 911bdd839d983af04003ffd0fc873824#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:24,900 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/dd040ebba0aa46be8a7f924056184611 is 50, key is test_row_0/C:col10/1731993863548/Put/seqid=0 2024-11-19T05:24:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741938_1114 (size=13425) 2024-11-19T05:24:24,934 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/dd040ebba0aa46be8a7f924056184611 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dd040ebba0aa46be8a7f924056184611 2024-11-19T05:24:24,941 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 911bdd839d983af04003ffd0fc873824/C of 911bdd839d983af04003ffd0fc873824 into dd040ebba0aa46be8a7f924056184611(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:24,942 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:24,942 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., storeName=911bdd839d983af04003ffd0fc873824/C, priority=13, startTime=1731993864829; duration=0sec 2024-11-19T05:24:24,942 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:24,942 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 911bdd839d983af04003ffd0fc873824:C 2024-11-19T05:24:25,111 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-19T05:24:25,111 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,111 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-19T05:24:25,111 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,112 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-19T05:24:25,112 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,115 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-19T05:24:25,115 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,116 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,116 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,117 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-19T05:24:25,360 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:25,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:25,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:24:25,752 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-19T05:24:25,752 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,752 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,752 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-19T05:24:25,753 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,753 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,753 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,753 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-19T05:24:25,753 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,753 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:25,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,753 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,753 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:25,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:25,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/3e138e3345b2438cbe530a154c4ffbe4 is 50, key is test_row_0/A:col10/1731993865748/Put/seqid=0 2024-11-19T05:24:25,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741939_1115 (size=12301) 2024-11-19T05:24:26,165 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/3e138e3345b2438cbe530a154c4ffbe4 2024-11-19T05:24:26,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/95b91fe62ff34e06ab38d1d9d623b911 is 50, key is test_row_0/B:col10/1731993865748/Put/seqid=0 2024-11-19T05:24:26,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741940_1116 (size=12301) 2024-11-19T05:24:26,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/95b91fe62ff34e06ab38d1d9d623b911 2024-11-19T05:24:26,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/3aaaf41d2e2447a08739a59789993a7f is 50, key is test_row_0/C:col10/1731993865748/Put/seqid=0 2024-11-19T05:24:26,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741941_1117 (size=12301) 2024-11-19T05:24:27,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=497 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/3aaaf41d2e2447a08739a59789993a7f 2024-11-19T05:24:27,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/3e138e3345b2438cbe530a154c4ffbe4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/3e138e3345b2438cbe530a154c4ffbe4 2024-11-19T05:24:27,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/3e138e3345b2438cbe530a154c4ffbe4, entries=150, sequenceid=497, filesize=12.0 K 2024-11-19T05:24:27,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/95b91fe62ff34e06ab38d1d9d623b911 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/95b91fe62ff34e06ab38d1d9d623b911 2024-11-19T05:24:27,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/95b91fe62ff34e06ab38d1d9d623b911, entries=150, sequenceid=497, filesize=12.0 K 2024-11-19T05:24:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/3aaaf41d2e2447a08739a59789993a7f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/3aaaf41d2e2447a08739a59789993a7f 2024-11-19T05:24:27,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/3aaaf41d2e2447a08739a59789993a7f, entries=150, sequenceid=497, filesize=12.0 K 2024-11-19T05:24:27,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 911bdd839d983af04003ffd0fc873824 in 1280ms, sequenceid=497, compaction requested=false 2024-11-19T05:24:27,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 911bdd839d983af04003ffd0fc873824: 2024-11-19T05:24:27,115 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:24:29,251 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-19T05:24:29,251 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:29,251 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:29,251 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:29,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:33,364 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-19T05:24:33,364 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:33,364 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:33,364 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:33,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 95 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3269 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3254 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1475 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4420 rows 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1476 2024-11-19T05:24:33,365 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4426 rows 2024-11-19T05:24:33,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:24:33,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:24:33,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:33,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:33,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:33,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:24:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:33,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:33,378 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993873378"}]},"ts":"1731993873378"} 2024-11-19T05:24:33,380 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:24:33,380 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:24:33,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:24:33,387 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, UNASSIGN}] 2024-11-19T05:24:33,388 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, UNASSIGN 2024-11-19T05:24:33,389 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=911bdd839d983af04003ffd0fc873824, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:33,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, UNASSIGN because future has completed 2024-11-19T05:24:33,391 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:24:33,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:33,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:33,550 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:33,550 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:24:33,551 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 911bdd839d983af04003ffd0fc873824, disabling compactions & flushes 2024-11-19T05:24:33,551 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:33,551 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:33,551 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. after waiting 0 ms 2024-11-19T05:24:33,551 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:33,552 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 911bdd839d983af04003ffd0fc873824 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T05:24:33,559 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5c34919d18554dd8a539a6dae2e18cea is 50, key is test_row_1/A:col10/1731993873359/Put/seqid=0 2024-11-19T05:24:33,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741942_1118 (size=9857) 2024-11-19T05:24:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:33,966 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5c34919d18554dd8a539a6dae2e18cea 2024-11-19T05:24:33,982 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/8111b34d9b024c0280172d658d19be1c is 50, key is test_row_1/B:col10/1731993873359/Put/seqid=0 2024-11-19T05:24:33,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741943_1119 (size=9857) 2024-11-19T05:24:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:34,389 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/8111b34d9b024c0280172d658d19be1c 2024-11-19T05:24:34,404 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 is 50, key is test_row_1/C:col10/1731993873359/Put/seqid=0 2024-11-19T05:24:34,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741944_1120 (size=9857) 2024-11-19T05:24:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:34,810 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 2024-11-19T05:24:34,821 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/A/5c34919d18554dd8a539a6dae2e18cea as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5c34919d18554dd8a539a6dae2e18cea 2024-11-19T05:24:34,828 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5c34919d18554dd8a539a6dae2e18cea, entries=100, sequenceid=502, filesize=9.6 K 2024-11-19T05:24:34,830 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/B/8111b34d9b024c0280172d658d19be1c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8111b34d9b024c0280172d658d19be1c 2024-11-19T05:24:34,835 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8111b34d9b024c0280172d658d19be1c, entries=100, sequenceid=502, filesize=9.6 K 2024-11-19T05:24:34,836 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/.tmp/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 2024-11-19T05:24:34,842 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/4bd0ce9a075b4b19b745f8b3e3ff13e9, entries=100, sequenceid=502, filesize=9.6 K 2024-11-19T05:24:34,843 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 911bdd839d983af04003ffd0fc873824 in 1292ms, sequenceid=502, compaction requested=true 2024-11-19T05:24:34,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/c1f13a7bf7e4452db027dbca64622111, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5d328a5ba3574ea28e3188cd3358771f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4e9d11444e3446039f910c64264431db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/311243f1ec0e4e3db6b350cdb38070e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e5d3de2664134093b604650cd061d318, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f02d618e95994f9197653a1411b60199, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/41b16f0efa304463b96a417c800a66c9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/95837926ae66404593ffaf35719abbe3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f0d9796b52124d479c7c2fdd68987f2c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d] to archive 2024-11-19T05:24:34,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:24:34,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4a508b78b4f8426b8bb796e610edd41e 2024-11-19T05:24:34,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ea3b854d63114cee926532426361f981 2024-11-19T05:24:34,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/c1f13a7bf7e4452db027dbca64622111 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/c1f13a7bf7e4452db027dbca64622111 2024-11-19T05:24:34,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/6655b022248c43b686699f9157766aa1 2024-11-19T05:24:34,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e011ae6cb7a34ef4a9789c06a86d730c 2024-11-19T05:24:34,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0b1ed8c71e314a30835ec9194e61974a 2024-11-19T05:24:34,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5d328a5ba3574ea28e3188cd3358771f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5d328a5ba3574ea28e3188cd3358771f 2024-11-19T05:24:34,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/d54645942bb243a0afe3bac69765e0d7 2024-11-19T05:24:34,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/fb292aa3e7a342acb5914ed83bdd7e59 2024-11-19T05:24:34,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f5ecb2b51ac44a2b967d07ec2301684b 2024-11-19T05:24:34,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4e9d11444e3446039f910c64264431db to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4e9d11444e3446039f910c64264431db 2024-11-19T05:24:34,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/0df961a848fb4284ba82d49ca0ff2d0a 2024-11-19T05:24:34,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8cb503849e1c4d76a9d663149dab73f0 2024-11-19T05:24:34,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/311243f1ec0e4e3db6b350cdb38070e0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/311243f1ec0e4e3db6b350cdb38070e0 2024-11-19T05:24:34,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/73ab3fdb36e745229981da55c5829c12 2024-11-19T05:24:34,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e5d3de2664134093b604650cd061d318 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/e5d3de2664134093b604650cd061d318 2024-11-19T05:24:34,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a674a0f1a7374123a834d5ffdba91a60 2024-11-19T05:24:34,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/933c5e7cfafb452c9d06d2c3e6bbf069 2024-11-19T05:24:34,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/ccf6b6561af8481ebb2e3c6f35aa5852 2024-11-19T05:24:34,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f02d618e95994f9197653a1411b60199 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f02d618e95994f9197653a1411b60199 2024-11-19T05:24:34,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/7ba0be9721754078975f266909c3c2b4 2024-11-19T05:24:34,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/29bb2035aae34ffd8bae9c25eca79667 2024-11-19T05:24:34,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/8aea1dbd1e4740ce9ddf41d6690cedca 2024-11-19T05:24:34,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/41b16f0efa304463b96a417c800a66c9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/41b16f0efa304463b96a417c800a66c9 2024-11-19T05:24:34,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/4ffb9602d8d042daab89f22720f57480 2024-11-19T05:24:34,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9244082c9dbe48b2b257407f6033b3bb 2024-11-19T05:24:34,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/95837926ae66404593ffaf35719abbe3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/95837926ae66404593ffaf35719abbe3 2024-11-19T05:24:34,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/64d1cde35b3a4d3893b29cc45af2a2f5 2024-11-19T05:24:34,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5213db4ad30b4ac497fa86fa22c1ca93 2024-11-19T05:24:34,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/79e3daee3a2a4277a7057d04a35805cb 2024-11-19T05:24:34,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f0d9796b52124d479c7c2fdd68987f2c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/f0d9796b52124d479c7c2fdd68987f2c 2024-11-19T05:24:34,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/71257bef8cae4687afacceda197e6598 2024-11-19T05:24:34,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/9bd977716d0249dba26f82480032612d 2024-11-19T05:24:34,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/57080d506ef7418ba23046b42e9cf24f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8ae12b8d84724acbbe856ad941cef9b3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/b03a2e20a92f4e3eac110fee90b18709, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/9aeff4426c6944afbf4e542a1a006f29, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1977aa5479484ad79e5da70b93139fc1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0614001428a7468b8433d4477a207a4a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9106d2875c7445685df6e88e923b685, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/d2f54e506eb1402b8dddc466479f4e5b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/329c76685d8046498a6e2223939dc939, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c] to archive 2024-11-19T05:24:34,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:24:34,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/2c3961b7d5134ab0a516138a25188d02 2024-11-19T05:24:34,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/59e8f11f9b8545cabb1d476db05ab92a 2024-11-19T05:24:34,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/57080d506ef7418ba23046b42e9cf24f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/57080d506ef7418ba23046b42e9cf24f 2024-11-19T05:24:34,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/451ff887c5fb4fdcabddd4d789eb8a18 2024-11-19T05:24:34,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/573dcf269caa45139c29fb2b771fdc6c 2024-11-19T05:24:34,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8ae12b8d84724acbbe856ad941cef9b3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8ae12b8d84724acbbe856ad941cef9b3 2024-11-19T05:24:34,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7be0bbe2047c49e2bf608d8995f70f77 2024-11-19T05:24:34,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/88447e3c9d154a33a6bf55190b7fbf9a 2024-11-19T05:24:34,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/411681c6544f4cd9a4f1608433b441ac 2024-11-19T05:24:34,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/b03a2e20a92f4e3eac110fee90b18709 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/b03a2e20a92f4e3eac110fee90b18709 2024-11-19T05:24:34,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7cd12eea96cd4b97aa818df1fd9cde04 2024-11-19T05:24:34,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/98a78c09d03042b99d6c223815221850 2024-11-19T05:24:34,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/9aeff4426c6944afbf4e542a1a006f29 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/9aeff4426c6944afbf4e542a1a006f29 2024-11-19T05:24:34,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e0e131a2e1b34824bf20b6c73d8f718f 2024-11-19T05:24:34,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/850af3ddd40a4b49ba91f7b8f45e4346 2024-11-19T05:24:34,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1977aa5479484ad79e5da70b93139fc1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1977aa5479484ad79e5da70b93139fc1 2024-11-19T05:24:34,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/05674e37d53e4c90967ee9cba2a92fda 2024-11-19T05:24:34,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/3097a1d03e5d4153a6aa57eea7038c01 2024-11-19T05:24:34,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0614001428a7468b8433d4477a207a4a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0614001428a7468b8433d4477a207a4a 2024-11-19T05:24:34,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/7e6603fcd8ae4d41b9b080ecbf2361ae 2024-11-19T05:24:34,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f680689136c44885a94d1e8d3072e4b9 2024-11-19T05:24:34,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9d4c849b4774b6989eb347fee22ed27 2024-11-19T05:24:34,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9106d2875c7445685df6e88e923b685 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e9106d2875c7445685df6e88e923b685 2024-11-19T05:24:34,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e7eae5c231304927aec00de7341345de 2024-11-19T05:24:34,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c9df3beeee03499492746d85eb7f13ba 2024-11-19T05:24:34,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/d2f54e506eb1402b8dddc466479f4e5b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/d2f54e506eb1402b8dddc466479f4e5b 2024-11-19T05:24:34,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/c7795e4ab853492a83eadffc9057ef1b 2024-11-19T05:24:34,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/1d83f3c071434eaa920dab04c9fb9ffb 2024-11-19T05:24:34,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/f9c05fa55e56489aa85117a35e31f1a9 2024-11-19T05:24:34,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/329c76685d8046498a6e2223939dc939 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/329c76685d8046498a6e2223939dc939 2024-11-19T05:24:34,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/32cfca5498f345d4863e14075ffbc595 2024-11-19T05:24:34,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/e635c25577c54ddcab6cf8cbbe048cf1 2024-11-19T05:24:34,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/0584b1904d6948adbeebe0df1ea6ad9c 2024-11-19T05:24:34,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/beadca3fe89246b58a627df1928b0dac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/43c565056552414188834c43d102fd0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b7256f82e6fc4b0790b9926c6b6b03fd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/879b8ca96b984dcabd603beff2514813, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/ab79a71d2c934317b8b9adba07b1575d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dc643afc86704ed68a701f77b04f201b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d5757f9077644ba38c9d840cddd466e8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/acd7e6a45b85407dad916d96daef62ad, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/0bdce35ab33f4c39b013eac98c4b76cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc] to archive 2024-11-19T05:24:34,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:24:34,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b6c5d62f8bba427fb984b27d29cde018 2024-11-19T05:24:34,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/caf727f5ea7a428c98fc79275d6bd65d 2024-11-19T05:24:34,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/beadca3fe89246b58a627df1928b0dac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/beadca3fe89246b58a627df1928b0dac 2024-11-19T05:24:34,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79e86f71b2904a46a2c464687de47a8f 2024-11-19T05:24:34,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c98af6c078eb42a1913f3bf0913509cd 2024-11-19T05:24:34,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/43c565056552414188834c43d102fd0a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/43c565056552414188834c43d102fd0a 2024-11-19T05:24:34,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8b3202efd32041cfa1291b64bbe047f8 2024-11-19T05:24:34,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d7f322474dcb4d16b4e84a88e493330f 2024-11-19T05:24:34,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/6cc55fa4b729427f95925ce99978f7e6 2024-11-19T05:24:34,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b7256f82e6fc4b0790b9926c6b6b03fd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b7256f82e6fc4b0790b9926c6b6b03fd 2024-11-19T05:24:34,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/1d9bfb8b5e94411b922bb41574ca4e8c 2024-11-19T05:24:34,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/b41b71a71f0a40ec88e133a345fcaae6 2024-11-19T05:24:34,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/879b8ca96b984dcabd603beff2514813 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/879b8ca96b984dcabd603beff2514813 2024-11-19T05:24:34,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/18a760748b054bbf9a5ed2eb246bf5bb 2024-11-19T05:24:34,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/92fad9b980a54299884c2215b60e38c2 2024-11-19T05:24:34,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/ab79a71d2c934317b8b9adba07b1575d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/ab79a71d2c934317b8b9adba07b1575d 2024-11-19T05:24:34,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/38268a009fee4688a6ce5f096a5bdd6a 2024-11-19T05:24:34,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/31f4be6378674a0dbe7f28a2450ec9db 2024-11-19T05:24:34,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dc643afc86704ed68a701f77b04f201b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dc643afc86704ed68a701f77b04f201b 2024-11-19T05:24:34,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/cf1003aaf90a4ff0994efc1601bd8892 2024-11-19T05:24:34,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/79bc4753f61146d7b9362bbf5359d072 2024-11-19T05:24:34,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/c373031459284e2d887e4163fa82f530 2024-11-19T05:24:34,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d5757f9077644ba38c9d840cddd466e8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/d5757f9077644ba38c9d840cddd466e8 2024-11-19T05:24:34,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/283a2f2f18c04dd3a0af978a5a56d484 2024-11-19T05:24:34,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/aead6a2003a8408188add36332152ca8 2024-11-19T05:24:34,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/acd7e6a45b85407dad916d96daef62ad to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/acd7e6a45b85407dad916d96daef62ad 2024-11-19T05:24:34,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/2cfbd1f29a004c7ba55de12baad0fb41 2024-11-19T05:24:34,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/9ce4fd2379464b29af18aa93b53e09c9 2024-11-19T05:24:34,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/8292532b58444d858694754e5ee5315b 2024-11-19T05:24:34,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/0bdce35ab33f4c39b013eac98c4b76cb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/0bdce35ab33f4c39b013eac98c4b76cb 2024-11-19T05:24:34,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/fd9e8b91ac2d49dcbdc31f410e032184 2024-11-19T05:24:34,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/510baa451f9e408ea0b5ac4c6e454781 2024-11-19T05:24:34,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/7ddd943dfe2b452da4e42f5703db67cc 2024-11-19T05:24:35,000 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/recovered.edits/505.seqid, newMaxSeqId=505, maxSeqId=1 2024-11-19T05:24:35,001 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. 2024-11-19T05:24:35,001 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 911bdd839d983af04003ffd0fc873824: Waiting for close lock at 1731993873551Running coprocessor pre-close hooks at 1731993873551Disabling compacts and flushes for region at 1731993873551Disabling writes for close at 1731993873551Obtaining lock to block concurrent updates at 1731993873552 (+1 ms)Preparing flush snapshotting stores in 911bdd839d983af04003ffd0fc873824 at 1731993873552Finished memstore snapshotting TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731993873552Flushing stores of TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824. at 1731993873553 (+1 ms)Flushing 911bdd839d983af04003ffd0fc873824/A: creating writer at 1731993873553Flushing 911bdd839d983af04003ffd0fc873824/A: appending metadata at 1731993873558 (+5 ms)Flushing 911bdd839d983af04003ffd0fc873824/A: closing flushed file at 1731993873558Flushing 911bdd839d983af04003ffd0fc873824/B: creating writer at 1731993873978 (+420 ms)Flushing 911bdd839d983af04003ffd0fc873824/B: appending metadata at 1731993873981 (+3 ms)Flushing 911bdd839d983af04003ffd0fc873824/B: closing flushed file at 1731993873981Flushing 911bdd839d983af04003ffd0fc873824/C: creating writer at 1731993874400 (+419 ms)Flushing 911bdd839d983af04003ffd0fc873824/C: appending metadata at 1731993874404 (+4 ms)Flushing 911bdd839d983af04003ffd0fc873824/C: closing flushed file at 1731993874404Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@521e4082: reopening flushed file at 1731993874819 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e12a5df: reopening flushed file at 1731993874829 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@165b2c1b: reopening flushed file at 1731993874835 (+6 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 911bdd839d983af04003ffd0fc873824 in 1292ms, sequenceid=502, compaction requested=true at 1731993874843 (+8 ms)Writing region close event to WAL at 1731993874996 (+153 ms)Running coprocessor post-close hooks at 1731993875000 (+4 ms)Closed at 1731993875000 2024-11-19T05:24:35,003 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:35,004 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=911bdd839d983af04003ffd0fc873824, regionState=CLOSED 2024-11-19T05:24:35,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:24:35,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-19T05:24:35,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 911bdd839d983af04003ffd0fc873824, server=f990f635ffd0,38449,1731993838792 in 1.6140 sec 2024-11-19T05:24:35,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-19T05:24:35,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=911bdd839d983af04003ffd0fc873824, UNASSIGN in 1.6210 sec 2024-11-19T05:24:35,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-19T05:24:35,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6290 sec 2024-11-19T05:24:35,014 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993875014"}]},"ts":"1731993875014"} 2024-11-19T05:24:35,015 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:24:35,015 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:24:35,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6440 sec 2024-11-19T05:24:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-19T05:24:35,528 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:35,532 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:24:35,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,538 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T05:24:35,541 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,545 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:35,551 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/recovered.edits] 2024-11-19T05:24:35,555 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/3e138e3345b2438cbe530a154c4ffbe4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/3e138e3345b2438cbe530a154c4ffbe4 2024-11-19T05:24:35,557 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5c34919d18554dd8a539a6dae2e18cea to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/5c34919d18554dd8a539a6dae2e18cea 2024-11-19T05:24:35,559 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a8704bab26644f2aa4a3fcb6cb895ca0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/A/a8704bab26644f2aa4a3fcb6cb895ca0 2024-11-19T05:24:35,562 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/4026f01e25fd455fb3e11893f78f69a3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/4026f01e25fd455fb3e11893f78f69a3 2024-11-19T05:24:35,563 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8111b34d9b024c0280172d658d19be1c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/8111b34d9b024c0280172d658d19be1c 2024-11-19T05:24:35,565 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/95b91fe62ff34e06ab38d1d9d623b911 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/B/95b91fe62ff34e06ab38d1d9d623b911 2024-11-19T05:24:35,569 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/3aaaf41d2e2447a08739a59789993a7f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/3aaaf41d2e2447a08739a59789993a7f 2024-11-19T05:24:35,571 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/4bd0ce9a075b4b19b745f8b3e3ff13e9 2024-11-19T05:24:35,574 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dd040ebba0aa46be8a7f924056184611 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/C/dd040ebba0aa46be8a7f924056184611 2024-11-19T05:24:35,579 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/recovered.edits/505.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824/recovered.edits/505.seqid 2024-11-19T05:24:35,580 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/911bdd839d983af04003ffd0fc873824 2024-11-19T05:24:35,580 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:24:35,588 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-19T05:24:35,601 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:24:35,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:24:35,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,635 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:24:35,635 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731993875635"}]},"ts":"9223372036854775807"} 2024-11-19T05:24:35,640 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:24:35,640 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 911bdd839d983af04003ffd0fc873824, NAME => 'TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:24:35,640 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:24:35,641 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731993875640"}]},"ts":"9223372036854775807"} 2024-11-19T05:24:35,644 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:24:35,645 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 113 msec 2024-11-19T05:24:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-19T05:24:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:24:35,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:35,661 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;f990f635ffd0:38449-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-299407974_22 at /127.0.0.1:44354 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 405), ProcessCount=11 (was 11), AvailableMemoryMB=12152 (was 13340) 2024-11-19T05:24:35,670 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=12152 2024-11-19T05:24:35,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:24:35,672 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:24:35,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:35,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:24:35,674 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:35,674 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-19T05:24:35,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:35,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:24:35,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741945_1121 (size=959) 2024-11-19T05:24:35,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:35,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:36,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:24:36,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741946_1122 (size=53) 2024-11-19T05:24:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:36,493 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:36,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a0658358bd5bb50671391f76ab5b1db1, disabling compactions & flushes 2024-11-19T05:24:36,494 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. after waiting 0 ms 2024-11-19T05:24:36,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,494 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,494 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a0658358bd5bb50671391f76ab5b1db1: Waiting for close lock at 1731993876494Disabling compacts and flushes for region at 1731993876494Disabling writes for close at 1731993876494Writing region close event to WAL at 1731993876494Closed at 1731993876494 2024-11-19T05:24:36,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:24:36,498 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993876497"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993876497"}]},"ts":"1731993876497"} 2024-11-19T05:24:36,500 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:24:36,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:24:36,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993876502"}]},"ts":"1731993876502"} 2024-11-19T05:24:36,505 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:24:36,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, ASSIGN}] 2024-11-19T05:24:36,508 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, ASSIGN 2024-11-19T05:24:36,509 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:24:36,659 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:36,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, ASSIGN because future has completed 2024-11-19T05:24:36,663 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:36,821 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,821 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:24:36,822 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,822 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:36,822 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,822 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,825 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,829 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName A 2024-11-19T05:24:36,829 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:36,831 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:36,831 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,833 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName B 2024-11-19T05:24:36,834 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:36,835 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:36,835 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,836 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName C 2024-11-19T05:24:36,836 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:36,837 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:36,837 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,838 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,838 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,839 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,839 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,840 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:24:36,841 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,843 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:24:36,843 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened a0658358bd5bb50671391f76ab5b1db1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70334333, jitterRate=0.048063233494758606}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:24:36,843 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:36,844 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for a0658358bd5bb50671391f76ab5b1db1: Running coprocessor pre-open hook at 1731993876822Writing region info on filesystem at 1731993876822Initializing all the Stores at 1731993876824 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993876824Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993876824Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993876824Cleaning up temporary data from old regions at 1731993876839 (+15 ms)Running coprocessor post-open hooks at 1731993876843 (+4 ms)Region opened successfully at 1731993876844 (+1 ms) 2024-11-19T05:24:36,845 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., pid=30, masterSystemTime=1731993876816 2024-11-19T05:24:36,847 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,847 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:36,848 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:36,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:24:36,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-19T05:24:36,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 in 188 msec 2024-11-19T05:24:36,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-19T05:24:36,855 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, ASSIGN in 347 msec 2024-11-19T05:24:36,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:24:36,856 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993876856"}]},"ts":"1731993876856"} 2024-11-19T05:24:36,857 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:24:36,858 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:24:36,859 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-19T05:24:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-19T05:24:37,818 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:37,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffee9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:37,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:37,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:37,822 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:37,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2df6a029, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:37,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:37,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:37,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:37,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f00002a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:37,826 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:37,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:37,831 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:24:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:24:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@427eb97b 2024-11-19T05:24:37,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:24:37,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:24:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:24:37,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:24:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T05:24:37,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741947_1123 (size=995) 2024-11-19T05:24:38,261 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T05:24:38,261 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T05:24:38,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:24:38,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE}] 2024-11-19T05:24:38,290 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE 2024-11-19T05:24:38,291 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:38,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE because future has completed 2024-11-19T05:24:38,295 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:24:38,295 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:38,448 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,448 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:24:38,448 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing a0658358bd5bb50671391f76ab5b1db1, disabling compactions & flushes 2024-11-19T05:24:38,448 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,448 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,449 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. after waiting 0 ms 2024-11-19T05:24:38,449 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,453 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T05:24:38,454 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,454 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for a0658358bd5bb50671391f76ab5b1db1: Waiting for close lock at 1731993878448Running coprocessor pre-close hooks at 1731993878448Disabling compacts and flushes for region at 1731993878448Disabling writes for close at 1731993878449 (+1 ms)Writing region close event to WAL at 1731993878449Running coprocessor post-close hooks at 1731993878454 (+5 ms)Closed at 1731993878454 2024-11-19T05:24:38,454 WARN [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: a0658358bd5bb50671391f76ab5b1db1 to self. 2024-11-19T05:24:38,456 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,457 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=CLOSED 2024-11-19T05:24:38,460 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:24:38,468 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-19T05:24:38,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 in 170 msec 2024-11-19T05:24:38,469 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE; state=CLOSED, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=true 2024-11-19T05:24:38,619 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:38,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE because future has completed 2024-11-19T05:24:38,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:24:38,778 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,779 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:24:38,779 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,779 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:24:38,780 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,780 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,785 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,793 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName A 2024-11-19T05:24:38,795 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:38,795 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:38,796 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,797 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName B 2024-11-19T05:24:38,797 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:38,797 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:38,797 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,798 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a0658358bd5bb50671391f76ab5b1db1 columnFamilyName C 2024-11-19T05:24:38,798 DEBUG [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:38,798 INFO [StoreOpener-a0658358bd5bb50671391f76ab5b1db1-1 {}] regionserver.HStore(327): Store=a0658358bd5bb50671391f76ab5b1db1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:24:38,798 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,799 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,800 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,801 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,801 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,802 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:24:38,803 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,804 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened a0658358bd5bb50671391f76ab5b1db1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62901573, jitterRate=-0.0626935213804245}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:24:38,804 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,805 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for a0658358bd5bb50671391f76ab5b1db1: Running coprocessor pre-open hook at 1731993878780Writing region info on filesystem at 1731993878780Initializing all the Stores at 1731993878782 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993878782Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993878782Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993878783 (+1 ms)Cleaning up temporary data from old regions at 1731993878801 (+18 ms)Running coprocessor post-open hooks at 1731993878804 (+3 ms)Region opened successfully at 1731993878805 (+1 ms) 2024-11-19T05:24:38,806 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., pid=35, masterSystemTime=1731993878773 2024-11-19T05:24:38,808 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,808 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:38,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=OPEN, openSeqNum=5, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:38,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:24:38,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-19T05:24:38,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 in 190 msec 2024-11-19T05:24:38,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-19T05:24:38,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, REOPEN/MOVE in 525 msec 2024-11-19T05:24:38,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-19T05:24:38,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 551 msec 2024-11-19T05:24:38,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 976 msec 2024-11-19T05:24:38,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-19T05:24:38,823 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:38,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f683619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,824 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ad9fd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,825 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,826 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59c6bede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd47c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,828 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,829 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2788175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@750c1845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,831 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,831 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5602194c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,833 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1da759, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,834 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85a5f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,834 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,834 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,835 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ad9c13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,836 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cdeb5a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,838 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42195826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,840 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@906917d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ab1fb6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,842 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69954a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,843 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4942d310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3880fed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,847 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,848 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f62ce7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,849 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73ebdaf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,850 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538759e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:24:38,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:24:38,852 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:24:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:24:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:24:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@417855e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:24:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:24:38,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:38,853 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:24:38,854 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13aa3d06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:24:38,855 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,855 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,855 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,855 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:38,855 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,856 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,856 DEBUG [Thread-634 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,856 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,856 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,856 DEBUG [Thread-634 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,857 DEBUG [Thread-621 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:24:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-19T05:24:38,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:24:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,863 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:24:38,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,864 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:38,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:24:38,866 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:38,867 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:38,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:24:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:38,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:24:38,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:38,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:38,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119df2fabc2475a42549374c9cbf84288c4_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993878928/Put/seqid=0 2024-11-19T05:24:39,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993939010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993939012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993939013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993939015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993939015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741948_1124 (size=12154) 2024-11-19T05:24:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993939129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993939130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993939130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993939130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993939129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:39,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:39,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993939337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993939338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993939338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993939338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993939347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:39,446 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119df2fabc2475a42549374c9cbf84288c4_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119df2fabc2475a42549374c9cbf84288c4_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:39,448 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a9d93019f98c4149aa0d1298edf309d4, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:39,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a9d93019f98c4149aa0d1298edf309d4 is 175, key is test_row_0/A:col10/1731993878928/Put/seqid=0 2024-11-19T05:24:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:39,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741949_1125 (size=30955) 2024-11-19T05:24:39,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a9d93019f98c4149aa0d1298edf309d4 2024-11-19T05:24:39,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/15c0089191cc465cb7db18d574f800ce is 50, key is test_row_0/B:col10/1731993878928/Put/seqid=0 2024-11-19T05:24:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741950_1126 (size=12001) 2024-11-19T05:24:39,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/15c0089191cc465cb7db18d574f800ce 2024-11-19T05:24:39,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/a1896f0005a645eda94d128011c304de is 50, key is test_row_0/C:col10/1731993878928/Put/seqid=0 2024-11-19T05:24:39,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741951_1127 (size=12001) 2024-11-19T05:24:39,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/a1896f0005a645eda94d128011c304de 2024-11-19T05:24:39,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:39,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993939657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a9d93019f98c4149aa0d1298edf309d4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993939658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993939658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993939658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4, entries=150, sequenceid=16, filesize=30.2 K 2024-11-19T05:24:39,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993939658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:39,673 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:39,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/15c0089191cc465cb7db18d574f800ce as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce 2024-11-19T05:24:39,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:24:39,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/a1896f0005a645eda94d128011c304de as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de 2024-11-19T05:24:39,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:24:39,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for a0658358bd5bb50671391f76ab5b1db1 in 781ms, sequenceid=16, compaction requested=false 2024-11-19T05:24:39,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:39,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-19T05:24:39,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:39,807 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:24:39,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8edf41b428840809d0ba66f4af82fd6_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993879000/Put/seqid=0 2024-11-19T05:24:39,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741952_1128 (size=12154) 2024-11-19T05:24:39,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:39,885 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b8edf41b428840809d0ba66f4af82fd6_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8edf41b428840809d0ba66f4af82fd6_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:39,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0ac84bf42e7f4ec4b4ad2ed6ac276619, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:39,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 is 175, key is test_row_0/A:col10/1731993879000/Put/seqid=0 2024-11-19T05:24:39,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741953_1129 (size=30955) 2024-11-19T05:24:39,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 2024-11-19T05:24:39,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/4e160c4f9b5044e39cd2d6094d082863 is 50, key is test_row_0/B:col10/1731993879000/Put/seqid=0 2024-11-19T05:24:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:40,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741954_1130 (size=12001) 2024-11-19T05:24:40,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:40,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993940176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993940176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993940178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993940179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993940181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993940287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993940287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993940298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993940298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,427 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/4e160c4f9b5044e39cd2d6094d082863 2024-11-19T05:24:40,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/448ae2e26d6d468eb1285088dd3306a4 is 50, key is test_row_0/C:col10/1731993879000/Put/seqid=0 2024-11-19T05:24:40,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741955_1131 (size=12001) 2024-11-19T05:24:40,459 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/448ae2e26d6d468eb1285088dd3306a4 2024-11-19T05:24:40,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 2024-11-19T05:24:40,473 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619, entries=150, sequenceid=41, filesize=30.2 K 2024-11-19T05:24:40,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/4e160c4f9b5044e39cd2d6094d082863 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863 2024-11-19T05:24:40,490 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:24:40,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/448ae2e26d6d468eb1285088dd3306a4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4 2024-11-19T05:24:40,499 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:24:40,504 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a0658358bd5bb50671391f76ab5b1db1 in 697ms, sequenceid=41, compaction requested=false 2024-11-19T05:24:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:40,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-19T05:24:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-19T05:24:40,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-19T05:24:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:40,507 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:40,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6390 sec 2024-11-19T05:24:40,507 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:24:40,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.6480 sec 2024-11-19T05:24:40,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111905aad005c45243a9b302bba4d92d525a_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:40,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741956_1132 (size=14594) 2024-11-19T05:24:40,563 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:24:40,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993940566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,579 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993940568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993940575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993940577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993940687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993940687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993940687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993940688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993940897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993940897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993940897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993940898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:40,955 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:40,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111905aad005c45243a9b302bba4d92d525a_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111905aad005c45243a9b302bba4d92d525a_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:40,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/bf5f904998884f5680a8ae0d44cd0c66, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:40,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/bf5f904998884f5680a8ae0d44cd0c66 is 175, key is test_row_0/A:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:40,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741957_1133 (size=39549) 2024-11-19T05:24:40,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/bf5f904998884f5680a8ae0d44cd0c66 2024-11-19T05:24:40,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/67f82743f43f45f09cbe4acf10dea0f0 is 50, key is test_row_0/B:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-19T05:24:41,008 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:41,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-19T05:24:41,011 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:41,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741958_1134 (size=12001) 2024-11-19T05:24:41,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/67f82743f43f45f09cbe4acf10dea0f0 2024-11-19T05:24:41,012 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:41,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:41,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b125aefc46b8434789d481188f247580 is 50, key is test_row_0/C:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:41,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741959_1135 (size=12001) 2024-11-19T05:24:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:41,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T05:24:41,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:41,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993941197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,201 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993941207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993941207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993941207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993941207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T05:24:41,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:41,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b125aefc46b8434789d481188f247580 2024-11-19T05:24:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/bf5f904998884f5680a8ae0d44cd0c66 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66 2024-11-19T05:24:41,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66, entries=200, sequenceid=53, filesize=38.6 K 2024-11-19T05:24:41,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T05:24:41,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/67f82743f43f45f09cbe4acf10dea0f0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0 2024-11-19T05:24:41,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:41,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:41,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T05:24:41,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b125aefc46b8434789d481188f247580 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580 2024-11-19T05:24:41,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580, entries=150, sequenceid=53, filesize=11.7 K 2024-11-19T05:24:41,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a0658358bd5bb50671391f76ab5b1db1 in 989ms, sequenceid=53, compaction requested=true 2024-11-19T05:24:41,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:41,496 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:41,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:41,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:41,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:41,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:41,496 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:41,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:41,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:41,500 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:41,500 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:41,501 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:41,501 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,501 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:41,501 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,501 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=99.1 K 2024-11-19T05:24:41,501 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.2 K 2024-11-19T05:24:41,501 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,501 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 15c0089191cc465cb7db18d574f800ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993878927 2024-11-19T05:24:41,501 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66] 2024-11-19T05:24:41,502 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9d93019f98c4149aa0d1298edf309d4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993878927 2024-11-19T05:24:41,502 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e160c4f9b5044e39cd2d6094d082863, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993879000 2024-11-19T05:24:41,503 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac84bf42e7f4ec4b4ad2ed6ac276619, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993879000 2024-11-19T05:24:41,503 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5f904998884f5680a8ae0d44cd0c66, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:41,503 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f82743f43f45f09cbe4acf10dea0f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:41,514 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:41,517 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#117 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:41,518 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/d40a30b094634fa9b4608559e72c04e9 is 50, key is test_row_0/B:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:41,524 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111956e27fa3a92446a0adc40bed1e84430e_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:41,529 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111956e27fa3a92446a0adc40bed1e84430e_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:41,529 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111956e27fa3a92446a0adc40bed1e84430e_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:41,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741961_1137 (size=4469) 2024-11-19T05:24:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741960_1136 (size=12104) 2024-11-19T05:24:41,557 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#118 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:41,558 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/c4277760a2c147b1ab134091d77728ce is 175, key is test_row_0/A:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:41,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741962_1138 (size=31058) 2024-11-19T05:24:41,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-19T05:24:41,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,629 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:24:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:41,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ca5cbbe4382414184cc8a2360537377_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993880573/Put/seqid=0 2024-11-19T05:24:41,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741963_1139 (size=12154) 2024-11-19T05:24:41,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:41,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993941733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993941733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993941734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993941737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993941847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993941848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993941848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993941857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:41,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:41,976 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/d40a30b094634fa9b4608559e72c04e9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/d40a30b094634fa9b4608559e72c04e9 2024-11-19T05:24:41,986 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into d40a30b094634fa9b4608559e72c04e9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:41,986 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:41,986 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993881496; duration=0sec 2024-11-19T05:24:41,986 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:41,986 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:41,986 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:41,988 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:41,988 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:41,988 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:41,988 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.2 K 2024-11-19T05:24:41,990 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a1896f0005a645eda94d128011c304de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993878927 2024-11-19T05:24:41,990 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 448ae2e26d6d468eb1285088dd3306a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993879000 2024-11-19T05:24:41,991 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b125aefc46b8434789d481188f247580, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:41,997 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/c4277760a2c147b1ab134091d77728ce as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce 2024-11-19T05:24:42,004 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into c4277760a2c147b1ab134091d77728ce(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:42,006 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#120 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:42,006 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:42,006 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993881496; duration=0sec 2024-11-19T05:24:42,007 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:42,007 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:42,007 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/56a47cb73fb44bafbe03cffffb98e91b is 50, key is test_row_0/C:col10/1731993880174/Put/seqid=0 2024-11-19T05:24:42,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741964_1140 (size=12104) 2024-11-19T05:24:42,030 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/56a47cb73fb44bafbe03cffffb98e91b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/56a47cb73fb44bafbe03cffffb98e91b 2024-11-19T05:24:42,039 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into 56a47cb73fb44bafbe03cffffb98e91b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:42,040 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:42,040 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993881496; duration=0sec 2024-11-19T05:24:42,040 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:42,040 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:42,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993942067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993942067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993942068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993942068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:42,122 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195ca5cbbe4382414184cc8a2360537377_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ca5cbbe4382414184cc8a2360537377_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:42,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/b6ec9ccc0b6f4409ab4de1d056d89d2d, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:42,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/b6ec9ccc0b6f4409ab4de1d056d89d2d is 175, key is test_row_0/A:col10/1731993880573/Put/seqid=0 2024-11-19T05:24:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741965_1141 (size=30955) 2024-11-19T05:24:42,156 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/b6ec9ccc0b6f4409ab4de1d056d89d2d 2024-11-19T05:24:42,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cda5598ea52140ea8ffc4976ed92f7c8 is 50, key is test_row_0/B:col10/1731993880573/Put/seqid=0 2024-11-19T05:24:42,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741966_1142 (size=12001) 2024-11-19T05:24:42,200 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cda5598ea52140ea8ffc4976ed92f7c8 2024-11-19T05:24:42,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b25b9e5b89794b19bb94fa14bebe45d2 is 50, key is test_row_0/C:col10/1731993880573/Put/seqid=0 2024-11-19T05:24:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741967_1143 (size=12001) 2024-11-19T05:24:42,269 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b25b9e5b89794b19bb94fa14bebe45d2 2024-11-19T05:24:42,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/b6ec9ccc0b6f4409ab4de1d056d89d2d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d 2024-11-19T05:24:42,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d, entries=150, sequenceid=77, filesize=30.2 K 2024-11-19T05:24:42,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cda5598ea52140ea8ffc4976ed92f7c8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8 2024-11-19T05:24:42,294 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:24:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b25b9e5b89794b19bb94fa14bebe45d2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2 2024-11-19T05:24:42,305 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:24:42,308 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for a0658358bd5bb50671391f76ab5b1db1 in 679ms, sequenceid=77, compaction requested=false 2024-11-19T05:24:42,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:42,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:42,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-19T05:24:42,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-19T05:24:42,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-19T05:24:42,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2980 sec 2024-11-19T05:24:42,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.3050 sec 2024-11-19T05:24:42,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:42,382 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:42,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:42,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fdd17e8adf7149e2b61b02a9427d46ac_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:42,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741968_1144 (size=17034) 2024-11-19T05:24:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993942454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993942454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993942454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993942461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993942567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993942567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993942567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993942567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993942777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993942777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993942777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993942785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:42,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:42,828 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:42,837 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fdd17e8adf7149e2b61b02a9427d46ac_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdd17e8adf7149e2b61b02a9427d46ac_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:42,839 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/98d73090acd144708b1bcc6812b18dbc, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:42,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/98d73090acd144708b1bcc6812b18dbc is 175, key is test_row_0/A:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:42,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741969_1145 (size=48139) 2024-11-19T05:24:42,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/98d73090acd144708b1bcc6812b18dbc 2024-11-19T05:24:42,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4533ba3c3a64615af462533cd7e091c is 50, key is test_row_0/B:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:42,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741970_1146 (size=12001) 2024-11-19T05:24:43,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993943088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993943088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993943088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,093 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993943097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-19T05:24:43,168 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:43,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:43,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-19T05:24:43,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:43,173 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:43,174 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:43,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:43,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993943207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,211 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:43,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T05:24:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4533ba3c3a64615af462533cd7e091c 2024-11-19T05:24:43,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/7ba114b6b30b4c27a4dcdc2df02318da is 50, key is test_row_0/C:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:43,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741971_1147 (size=12001) 2024-11-19T05:24:43,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T05:24:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:43,498 INFO [master/f990f635ffd0:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-19T05:24:43,498 INFO [master/f990f635ffd0:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-19T05:24:43,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993943597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993943598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993943607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:43,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993943607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:43,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:43,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T05:24:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:43,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T05:24:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:43,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/7ba114b6b30b4c27a4dcdc2df02318da 2024-11-19T05:24:43,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/98d73090acd144708b1bcc6812b18dbc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc 2024-11-19T05:24:43,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc, entries=250, sequenceid=94, filesize=47.0 K 2024-11-19T05:24:43,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4533ba3c3a64615af462533cd7e091c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c 2024-11-19T05:24:43,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T05:24:43,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/7ba114b6b30b4c27a4dcdc2df02318da as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da 2024-11-19T05:24:43,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da, entries=150, sequenceid=94, filesize=11.7 K 2024-11-19T05:24:43,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for a0658358bd5bb50671391f76ab5b1db1 in 1471ms, sequenceid=94, compaction requested=true 2024-11-19T05:24:43,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:43,854 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:43,855 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:43,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:43,855 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:43,855 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:43,856 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,856 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=107.6 K 2024-11-19T05:24:43,856 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,856 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc] 2024-11-19T05:24:43,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4277760a2c147b1ab134091d77728ce, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:43,858 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ec9ccc0b6f4409ab4de1d056d89d2d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993880562 2024-11-19T05:24:43,858 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:43,858 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:43,858 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,858 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/d40a30b094634fa9b4608559e72c04e9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.3 K 2024-11-19T05:24:43,859 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98d73090acd144708b1bcc6812b18dbc, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:43,859 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d40a30b094634fa9b4608559e72c04e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:43,860 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting cda5598ea52140ea8ffc4976ed92f7c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993880562 2024-11-19T05:24:43,860 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f4533ba3c3a64615af462533cd7e091c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:43,870 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:43,892 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119e03349d553184823a0a834b72e9aa6f6_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:43,898 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#127 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:43,899 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/bf55bad0e82242afa17a547254f477a3 is 50, key is test_row_0/B:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:43,902 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119e03349d553184823a0a834b72e9aa6f6_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:43,902 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119e03349d553184823a0a834b72e9aa6f6_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:43,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741972_1148 (size=12207) 2024-11-19T05:24:43,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741973_1149 (size=4469) 2024-11-19T05:24:43,941 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#126 average throughput is 0.35 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:43,941 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08 is 175, key is test_row_0/A:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:43,947 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/bf55bad0e82242afa17a547254f477a3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/bf55bad0e82242afa17a547254f477a3 2024-11-19T05:24:43,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-19T05:24:43,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,950 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:24:43,960 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into bf55bad0e82242afa17a547254f477a3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:43,960 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:43,960 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993883855; duration=0sec 2024-11-19T05:24:43,960 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:43,960 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:43,960 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:43,962 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:43,962 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:43,962 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:43,962 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/56a47cb73fb44bafbe03cffffb98e91b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.3 K 2024-11-19T05:24:43,963 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a47cb73fb44bafbe03cffffb98e91b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731993880170 2024-11-19T05:24:43,964 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b25b9e5b89794b19bb94fa14bebe45d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993880562 2024-11-19T05:24:43,964 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ba114b6b30b4c27a4dcdc2df02318da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:43,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741974_1150 (size=31161) 2024-11-19T05:24:43,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cd53804ad1ae46d8b0626ab54b6e3d03_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993882459/Put/seqid=0 2024-11-19T05:24:43,982 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08 2024-11-19T05:24:43,998 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into 2bfbd8dd11cb4cfa9e7d8b4a1a366d08(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:43,999 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:43,999 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993883854; duration=0sec 2024-11-19T05:24:43,999 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:43,999 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:43,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741975_1151 (size=12154) 2024-11-19T05:24:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:44,005 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119cd53804ad1ae46d8b0626ab54b6e3d03_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd53804ad1ae46d8b0626ab54b6e3d03_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:44,007 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:44,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/aee71dba586b4facacf86c17fa44d2ac, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:44,007 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/76bd20fc953e493c8ea3216bcc23c809 is 50, key is test_row_0/C:col10/1731993881726/Put/seqid=0 2024-11-19T05:24:44,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/aee71dba586b4facacf86c17fa44d2ac is 175, key is test_row_0/A:col10/1731993882459/Put/seqid=0 2024-11-19T05:24:44,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741977_1153 (size=30955) 2024-11-19T05:24:44,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741976_1152 (size=12207) 2024-11-19T05:24:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:44,428 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/aee71dba586b4facacf86c17fa44d2ac 2024-11-19T05:24:44,439 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/76bd20fc953e493c8ea3216bcc23c809 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/76bd20fc953e493c8ea3216bcc23c809 2024-11-19T05:24:44,450 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into 76bd20fc953e493c8ea3216bcc23c809(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:44,450 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:44,451 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993883855; duration=0sec 2024-11-19T05:24:44,451 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:44,451 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:44,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/75d4e04c3ed047259e95462b99cbc3f5 is 50, key is test_row_0/B:col10/1731993882459/Put/seqid=0 2024-11-19T05:24:44,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741978_1154 (size=12001) 2024-11-19T05:24:44,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:44,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993944633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993944633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993944635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993944636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,644 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993944747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993944747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993944747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993944747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,887 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/75d4e04c3ed047259e95462b99cbc3f5 2024-11-19T05:24:44,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/1bb6a2e5faab4608ae21aacf282af47b is 50, key is test_row_0/C:col10/1731993882459/Put/seqid=0 2024-11-19T05:24:44,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741979_1155 (size=12001) 2024-11-19T05:24:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993944957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993944957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993944957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993944957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:44,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993945267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993945267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993945267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993945267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,300 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/1bb6a2e5faab4608ae21aacf282af47b 2024-11-19T05:24:45,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/aee71dba586b4facacf86c17fa44d2ac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac 2024-11-19T05:24:45,311 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac, entries=150, sequenceid=116, filesize=30.2 K 2024-11-19T05:24:45,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/75d4e04c3ed047259e95462b99cbc3f5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5 2024-11-19T05:24:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:45,317 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T05:24:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/1bb6a2e5faab4608ae21aacf282af47b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b 2024-11-19T05:24:45,323 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T05:24:45,324 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for a0658358bd5bb50671391f76ab5b1db1 in 1375ms, sequenceid=116, compaction requested=false 2024-11-19T05:24:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-19T05:24:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-19T05:24:45,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-19T05:24:45,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2024-11-19T05:24:45,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.1590 sec 2024-11-19T05:24:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:45,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:45,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:24:45,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef8e04c7197f470f99f3ad116b1ec9e7_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:45,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741980_1156 (size=17284) 2024-11-19T05:24:45,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993945801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993945804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993945806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993945807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993945917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993945917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993945917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993945917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:45,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993946127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993946127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993946127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993946127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:46,202 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ef8e04c7197f470f99f3ad116b1ec9e7_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef8e04c7197f470f99f3ad116b1ec9e7_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:46,203 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/598a022a3a2f4cafaa91c7b933a32fc4, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:46,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/598a022a3a2f4cafaa91c7b933a32fc4 is 175, key is test_row_0/A:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:46,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741981_1157 (size=48389) 2024-11-19T05:24:46,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993946438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993946438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993946438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993946438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:46,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:46,668 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/598a022a3a2f4cafaa91c7b933a32fc4 2024-11-19T05:24:46,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a062a38fdf8e49319712ad9c8418fb01 is 50, key is test_row_0/B:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:46,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741982_1158 (size=12151) 2024-11-19T05:24:46,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a062a38fdf8e49319712ad9c8418fb01 2024-11-19T05:24:46,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3b626bc415ce4ff1b774e59b3f21ac72 is 50, key is test_row_0/C:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:46,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741983_1159 (size=12151) 2024-11-19T05:24:46,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3b626bc415ce4ff1b774e59b3f21ac72 2024-11-19T05:24:46,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/598a022a3a2f4cafaa91c7b933a32fc4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4 2024-11-19T05:24:46,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4, entries=250, sequenceid=136, filesize=47.3 K 2024-11-19T05:24:46,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a062a38fdf8e49319712ad9c8418fb01 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01 2024-11-19T05:24:46,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01, entries=150, sequenceid=136, filesize=11.9 K 2024-11-19T05:24:46,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3b626bc415ce4ff1b774e59b3f21ac72 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72 2024-11-19T05:24:46,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72, entries=150, sequenceid=136, filesize=11.9 K 2024-11-19T05:24:46,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a0658358bd5bb50671391f76ab5b1db1 in 970ms, sequenceid=136, compaction requested=true 2024-11-19T05:24:46,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:46,752 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:46,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:46,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:46,752 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:46,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:46,753 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:46,754 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:46,754 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:46,754 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=107.9 K 2024-11-19T05:24:46,754 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:46,754 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4] 2024-11-19T05:24:46,754 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:46,754 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:46,755 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:46,755 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/bf55bad0e82242afa17a547254f477a3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.5 K 2024-11-19T05:24:46,756 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bfbd8dd11cb4cfa9e7d8b4a1a366d08, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:46,756 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting bf55bad0e82242afa17a547254f477a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:46,760 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d4e04c3ed047259e95462b99cbc3f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993882450 2024-11-19T05:24:46,761 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee71dba586b4facacf86c17fa44d2ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993882450 2024-11-19T05:24:46,761 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a062a38fdf8e49319712ad9c8418fb01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:46,761 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 598a022a3a2f4cafaa91c7b933a32fc4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:46,778 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:46,779 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/70387ba6182a490a85e42237a674b4ac is 50, key is test_row_0/B:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:46,779 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:46,793 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119189c7eb92df740ffab6251a9c597be28_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:46,795 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119189c7eb92df740ffab6251a9c597be28_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:46,795 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119189c7eb92df740ffab6251a9c597be28_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:46,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741984_1160 (size=12459) 2024-11-19T05:24:46,829 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/70387ba6182a490a85e42237a674b4ac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/70387ba6182a490a85e42237a674b4ac 2024-11-19T05:24:46,835 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into 70387ba6182a490a85e42237a674b4ac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:46,835 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:46,835 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993886752; duration=0sec 2024-11-19T05:24:46,836 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:46,836 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:46,836 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:46,838 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:46,838 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:46,838 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:46,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741985_1161 (size=4469) 2024-11-19T05:24:46,838 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/76bd20fc953e493c8ea3216bcc23c809, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=35.5 K 2024-11-19T05:24:46,839 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 76bd20fc953e493c8ea3216bcc23c809, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731993881726 2024-11-19T05:24:46,841 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb6a2e5faab4608ae21aacf282af47b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993882450 2024-11-19T05:24:46,841 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b626bc415ce4ff1b774e59b3f21ac72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:46,853 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:46,854 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/cb2f73e02a1d40589a6208c8a73bf2fa is 50, key is test_row_0/C:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:46,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741986_1162 (size=12459) 2024-11-19T05:24:46,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:46,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:46,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:24:46,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111962bbad573e8e4072a7b738bda90e3f65_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993885796/Put/seqid=0 2024-11-19T05:24:46,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741987_1163 (size=14794) 2024-11-19T05:24:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993947006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993947007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993947009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993947010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993947117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993947127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993947127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993947128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993947226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,230 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,240 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:47,240 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ee68cbfe01e7481cac2ac39b3476d398 is 175, key is test_row_0/A:col10/1731993884630/Put/seqid=0 2024-11-19T05:24:47,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741988_1164 (size=31413) 2024-11-19T05:24:47,300 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/cb2f73e02a1d40589a6208c8a73bf2fa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/cb2f73e02a1d40589a6208c8a73bf2fa 2024-11-19T05:24:47,304 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into cb2f73e02a1d40589a6208c8a73bf2fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:47,304 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:47,304 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993886753; duration=0sec 2024-11-19T05:24:47,304 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:47,305 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:47,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993947327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-19T05:24:47,337 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:47,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-19T05:24:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:47,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:47,339 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:47,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:47,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993947337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993947337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993947337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,382 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:47,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111962bbad573e8e4072a7b738bda90e3f65_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962bbad573e8e4072a7b738bda90e3f65_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:47,391 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86691bbd12eb4a9495dd18ce9372fd82, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:47,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86691bbd12eb4a9495dd18ce9372fd82 is 175, key is test_row_0/A:col10/1731993885796/Put/seqid=0 2024-11-19T05:24:47,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741989_1165 (size=39749) 2024-11-19T05:24:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:47,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993947640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:47,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993947648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,653 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ee68cbfe01e7481cac2ac39b3476d398 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993947648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993947649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:47,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:47,663 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into ee68cbfe01e7481cac2ac39b3476d398(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:47,663 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:47,663 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993886752; duration=0sec 2024-11-19T05:24:47,663 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:47,664 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:47,800 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86691bbd12eb4a9495dd18ce9372fd82 2024-11-19T05:24:47,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/09b9a8619d524463a769ffcd829a42ae is 50, key is test_row_0/B:col10/1731993885796/Put/seqid=0 2024-11-19T05:24:47,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741990_1166 (size=12151) 2024-11-19T05:24:47,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:47,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:48,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:48,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:48,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:48,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:48,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993948147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993948157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:48,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993948167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993948168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:48,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/09b9a8619d524463a769ffcd829a42ae 2024-11-19T05:24:48,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b0308c4e30904256a1e747a050314826 is 50, key is test_row_0/C:col10/1731993885796/Put/seqid=0 2024-11-19T05:24:48,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741991_1167 (size=12151) 2024-11-19T05:24:48,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b0308c4e30904256a1e747a050314826 2024-11-19T05:24:48,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86691bbd12eb4a9495dd18ce9372fd82 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82 2024-11-19T05:24:48,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82, entries=200, sequenceid=157, filesize=38.8 K 2024-11-19T05:24:48,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/09b9a8619d524463a769ffcd829a42ae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae 2024-11-19T05:24:48,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T05:24:48,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/b0308c4e30904256a1e747a050314826 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826 2024-11-19T05:24:48,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T05:24:48,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a0658358bd5bb50671391f76ab5b1db1 in 1308ms, sequenceid=157, compaction requested=false 2024-11-19T05:24:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:48,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-19T05:24:48,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:48,265 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:24:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9f1e0c2ce574035a3a7e3983c67df62_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993886971/Put/seqid=0 2024-11-19T05:24:48,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741992_1168 (size=12304) 2024-11-19T05:24:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:48,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:48,703 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9f1e0c2ce574035a3a7e3983c67df62_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9f1e0c2ce574035a3a7e3983c67df62_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:48,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/cd347a20a5974260bb8b4964f435402c, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:48,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/cd347a20a5974260bb8b4964f435402c is 175, key is test_row_0/A:col10/1731993886971/Put/seqid=0 2024-11-19T05:24:48,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741993_1169 (size=31105) 2024-11-19T05:24:49,110 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/cd347a20a5974260bb8b4964f435402c 2024-11-19T05:24:49,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/49787534725a4539a253832b00e5af71 is 50, key is test_row_0/B:col10/1731993886971/Put/seqid=0 2024-11-19T05:24:49,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741994_1170 (size=12151) 2024-11-19T05:24:49,133 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/49787534725a4539a253832b00e5af71 2024-11-19T05:24:49,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/262284eb5c444f31904065e1831ddd66 is 50, key is test_row_0/C:col10/1731993886971/Put/seqid=0 2024-11-19T05:24:49,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741995_1171 (size=12151) 2024-11-19T05:24:49,158 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/262284eb5c444f31904065e1831ddd66 2024-11-19T05:24:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/cd347a20a5974260bb8b4964f435402c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c 2024-11-19T05:24:49,168 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c, entries=150, sequenceid=175, filesize=30.4 K 2024-11-19T05:24:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:49,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/49787534725a4539a253832b00e5af71 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71 2024-11-19T05:24:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T05:24:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/262284eb5c444f31904065e1831ddd66 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66 2024-11-19T05:24:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,185 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T05:24:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,186 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=13.42 KB/13740 for a0658358bd5bb50671391f76ab5b1db1 in 921ms, sequenceid=175, compaction requested=true 2024-11-19T05:24:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-19T05:24:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-19T05:24:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-19T05:24:49,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-19T05:24:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8550 sec 2024-11-19T05:24:49,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:49,230 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:49,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:49,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111968998536cff140b28b5fc1d724ac8cfc_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741996_1172 (size=24758) 2024-11-19T05:24:49,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:49,270 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111968998536cff140b28b5fc1d724ac8cfc_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968998536cff140b28b5fc1d724ac8cfc_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:49,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/77715c06466c4b7488c181197f2bfabe, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:49,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/77715c06466c4b7488c181197f2bfabe is 175, key is test_row_0/A:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:49,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741997_1173 (size=74394) 2024-11-19T05:24:49,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993949289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993949289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993949298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993949298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,306 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993949408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993949408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993949408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993949417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-19T05:24:49,487 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:49,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-19T05:24:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:49,493 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:49,494 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:49,494 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993949617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993949617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993949617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993949627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=186, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/77715c06466c4b7488c181197f2bfabe 2024-11-19T05:24:49,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4de5a7532ff415bb230db595cb35286 is 50, key is test_row_0/B:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:49,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741998_1174 (size=12151) 2024-11-19T05:24:49,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:49,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:49,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:49,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993949927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993949927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993949927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993949937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:49,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:49,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:49,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:50,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4de5a7532ff415bb230db595cb35286 2024-11-19T05:24:50,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/e1ec65d5d7e44469be485338c9b3fecb is 50, key is test_row_0/C:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:50,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741999_1175 (size=12151) 2024-11-19T05:24:50,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993950437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:50,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993950437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:50,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993950437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:50,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993950447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:50,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/e1ec65d5d7e44469be485338c9b3fecb 2024-11-19T05:24:50,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/77715c06466c4b7488c181197f2bfabe as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe 2024-11-19T05:24:50,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe, entries=400, sequenceid=186, filesize=72.7 K 2024-11-19T05:24:50,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f4de5a7532ff415bb230db595cb35286 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286 2024-11-19T05:24:50,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T05:24:50,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/e1ec65d5d7e44469be485338c9b3fecb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb 2024-11-19T05:24:50,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb, entries=150, sequenceid=186, filesize=11.9 K 2024-11-19T05:24:50,566 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for a0658358bd5bb50671391f76ab5b1db1 in 1335ms, sequenceid=186, compaction requested=true 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:50,566 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:50,566 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:50,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:50,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176661 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:50,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:50,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=172.5 K 2024-11-19T05:24:50,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe] 2024-11-19T05:24:50,569 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:50,569 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:50,569 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,569 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/70387ba6182a490a85e42237a674b4ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=47.8 K 2024-11-19T05:24:50,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-19T05:24:50,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,569 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T05:24:50,571 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 70387ba6182a490a85e42237a674b4ac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:50,572 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b9a8619d524463a769ffcd829a42ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731993885796 2024-11-19T05:24:50,573 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 49787534725a4539a253832b00e5af71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993886971 2024-11-19T05:24:50,573 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f4de5a7532ff415bb230db595cb35286, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889170 2024-11-19T05:24:50,576 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee68cbfe01e7481cac2ac39b3476d398, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:50,577 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86691bbd12eb4a9495dd18ce9372fd82, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731993885796 2024-11-19T05:24:50,577 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd347a20a5974260bb8b4964f435402c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993886971 2024-11-19T05:24:50,578 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77715c06466c4b7488c181197f2bfabe, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889168 2024-11-19T05:24:50,588 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:50,588 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/64117a79f91b4ffc9883ff955b096e08 is 50, key is test_row_0/B:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:50,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191e095605de1b4081a2bd39030c4defe2_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993889287/Put/seqid=0 2024-11-19T05:24:50,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742000_1176 (size=12595) 2024-11-19T05:24:50,605 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:50,610 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119aa5e505164894d8f8722028b2b327ef2_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:50,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742001_1177 (size=12304) 2024-11-19T05:24:50,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,614 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119aa5e505164894d8f8722028b2b327ef2_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:50,614 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119aa5e505164894d8f8722028b2b327ef2_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:50,614 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/64117a79f91b4ffc9883ff955b096e08 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/64117a79f91b4ffc9883ff955b096e08 2024-11-19T05:24:50,620 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191e095605de1b4081a2bd39030c4defe2_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e095605de1b4081a2bd39030c4defe2_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:50,621 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into 64117a79f91b4ffc9883ff955b096e08(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:50,621 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:50,621 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=12, startTime=1731993890566; duration=0sec 2024-11-19T05:24:50,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/5347ca807a9c40aab4c70eb59932024c, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:50,621 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:50,621 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:50,622 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:24:50,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/5347ca807a9c40aab4c70eb59932024c is 175, key is test_row_0/A:col10/1731993889287/Put/seqid=0 2024-11-19T05:24:50,624 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:24:50,624 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:50,624 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,625 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/cb2f73e02a1d40589a6208c8a73bf2fa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=47.8 K 2024-11-19T05:24:50,625 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2f73e02a1d40589a6208c8a73bf2fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731993884630 2024-11-19T05:24:50,626 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b0308c4e30904256a1e747a050314826, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731993885796 2024-11-19T05:24:50,626 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 262284eb5c444f31904065e1831ddd66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993886971 2024-11-19T05:24:50,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742002_1178 (size=4469) 2024-11-19T05:24:50,627 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ec65d5d7e44469be485338c9b3fecb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889170 2024-11-19T05:24:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:50,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742003_1179 (size=31105) 2024-11-19T05:24:50,628 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#149 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:50,629 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/8cfe460eca8d4ef191389f313813fb83 is 175, key is test_row_0/A:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:50,629 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/5347ca807a9c40aab4c70eb59932024c 2024-11-19T05:24:50,646 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:50,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 is 50, key is test_row_0/B:col10/1731993889287/Put/seqid=0 2024-11-19T05:24:50,647 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/e57bb4e310a048d1971e742c8d7afcae is 50, key is test_row_0/C:col10/1731993889170/Put/seqid=0 2024-11-19T05:24:50,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742004_1180 (size=31549) 2024-11-19T05:24:50,704 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/8cfe460eca8d4ef191389f313813fb83 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83 2024-11-19T05:24:50,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742005_1181 (size=12595) 2024-11-19T05:24:50,714 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into 8cfe460eca8d4ef191389f313813fb83(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:50,714 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:50,714 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=12, startTime=1731993890566; duration=0sec 2024-11-19T05:24:50,714 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:50,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742006_1182 (size=12151) 2024-11-19T05:24:50,715 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:50,716 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 2024-11-19T05:24:50,716 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/e57bb4e310a048d1971e742c8d7afcae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e57bb4e310a048d1971e742c8d7afcae 2024-11-19T05:24:50,726 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into e57bb4e310a048d1971e742c8d7afcae(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:50,726 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:50,726 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=12, startTime=1731993890566; duration=0sec 2024-11-19T05:24:50,726 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:50,726 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:50,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/8f419478754747618ed17f3f142d89ba is 50, key is test_row_0/C:col10/1731993889287/Put/seqid=0 2024-11-19T05:24:50,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742007_1183 (size=12151) 2024-11-19T05:24:50,746 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/8f419478754747618ed17f3f142d89ba 2024-11-19T05:24:50,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/5347ca807a9c40aab4c70eb59932024c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c 2024-11-19T05:24:50,768 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c, entries=150, sequenceid=212, filesize=30.4 K 2024-11-19T05:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 2024-11-19T05:24:50,774 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T05:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/8f419478754747618ed17f3f142d89ba as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba 2024-11-19T05:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba, entries=150, sequenceid=212, filesize=11.9 K 2024-11-19T05:24:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,784 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for a0658358bd5bb50671391f76ab5b1db1 in 215ms, sequenceid=212, compaction requested=false 2024-11-19T05:24:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-19T05:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-19T05:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-19T05:24:50,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2920 sec 2024-11-19T05:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2990 sec 2024-11-19T05:24:50,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:50,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:51,491 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:51,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:24:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,505 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193968e7057f104ecca86c87922e2bc647_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742009_1185 (size=24758) 2024-11-19T05:24:51,528 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:51,535 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193968e7057f104ecca86c87922e2bc647_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193968e7057f104ecca86c87922e2bc647_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:51,536 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/e74983abb7f641389187fd666ce9b4fc, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:51,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/e74983abb7f641389187fd666ce9b4fc is 175, key is test_row_0/A:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:51,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742008_1184 (size=74395) 2024-11-19T05:24:51,541 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/e74983abb7f641389187fd666ce9b4fc 2024-11-19T05:24:51,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993951547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993951549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993951551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,558 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993951553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/b9952174cc0e48138ac52f385a50d345 is 50, key is test_row_0/B:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:51,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742010_1186 (size=12151) 2024-11-19T05:24:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-19T05:24:51,637 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:51,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:51,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993951657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993951657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-19T05:24:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:51,664 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:51,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:51,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993951667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993951668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:51,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:51,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:51,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993951867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993951868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993951878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993951878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:51,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:51,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:51,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:51,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:51,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:51,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:51,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/b9952174cc0e48138ac52f385a50d345 2024-11-19T05:24:52,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/485a2c8640d3472fa366cd285155bc51 is 50, key is test_row_0/C:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:52,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742011_1187 (size=12151) 2024-11-19T05:24:52,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993952177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993952187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993952187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993952187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:52,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/485a2c8640d3472fa366cd285155bc51 2024-11-19T05:24:52,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/e74983abb7f641389187fd666ce9b4fc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc 2024-11-19T05:24:52,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc, entries=400, sequenceid=226, filesize=72.7 K 2024-11-19T05:24:52,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/b9952174cc0e48138ac52f385a50d345 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345 2024-11-19T05:24:52,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T05:24:52,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/485a2c8640d3472fa366cd285155bc51 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51 2024-11-19T05:24:52,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:52,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51, entries=150, sequenceid=226, filesize=11.9 K 2024-11-19T05:24:52,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a0658358bd5bb50671391f76ab5b1db1 in 942ms, sequenceid=226, compaction requested=true 2024-11-19T05:24:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:52,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:52,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:52,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:52,434 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:52,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:24:52,434 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:52,435 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,435 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,435 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=133.8 K 2024-11-19T05:24:52,435 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/64117a79f91b4ffc9883ff955b096e08, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.0 K 2024-11-19T05:24:52,435 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc] 2024-11-19T05:24:52,435 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 64117a79f91b4ffc9883ff955b096e08, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889170 2024-11-19T05:24:52,436 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cfe460eca8d4ef191389f313813fb83, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889170 2024-11-19T05:24:52,436 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 47ebbeff1ee74bcd9a40fda5d1a4cac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731993889280 2024-11-19T05:24:52,436 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5347ca807a9c40aab4c70eb59932024c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731993889280 2024-11-19T05:24:52,436 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b9952174cc0e48138ac52f385a50d345, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891486 2024-11-19T05:24:52,437 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e74983abb7f641389187fd666ce9b4fc, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891474 2024-11-19T05:24:52,448 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:52,450 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411196a0d2366293543e7a8257e667704a187_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:52,450 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:52,451 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f3c024777dc549469bf9fd09810e8b06 is 50, key is test_row_0/B:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:52,453 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411196a0d2366293543e7a8257e667704a187_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:52,453 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196a0d2366293543e7a8257e667704a187_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:52,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742012_1188 (size=12697) 2024-11-19T05:24:52,468 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/f3c024777dc549469bf9fd09810e8b06 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f3c024777dc549469bf9fd09810e8b06 2024-11-19T05:24:52,477 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into f3c024777dc549469bf9fd09810e8b06(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:52,477 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:52,477 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993892433; duration=0sec 2024-11-19T05:24:52,477 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:52,477 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:52,478 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:52,480 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:52,480 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:52,480 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,480 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e57bb4e310a048d1971e742c8d7afcae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.0 K 2024-11-19T05:24:52,481 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e57bb4e310a048d1971e742c8d7afcae, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731993889170 2024-11-19T05:24:52,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742013_1189 (size=4469) 2024-11-19T05:24:52,483 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#156 average throughput is 0.70 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:52,483 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/add6e416ef6344f18e5ee82412c2cbeb is 175, key is test_row_0/A:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:52,484 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f419478754747618ed17f3f142d89ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731993889280 2024-11-19T05:24:52,485 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 485a2c8640d3472fa366cd285155bc51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891486 2024-11-19T05:24:52,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742014_1190 (size=31651) 2024-11-19T05:24:52,518 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:52,519 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/de039aca0f07452e816df60d6917b25e is 50, key is test_row_0/C:col10/1731993891489/Put/seqid=0 2024-11-19T05:24:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742015_1191 (size=12697) 2024-11-19T05:24:52,551 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/de039aca0f07452e816df60d6917b25e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de039aca0f07452e816df60d6917b25e 2024-11-19T05:24:52,568 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into de039aca0f07452e816df60d6917b25e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:52,568 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:52,568 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993892434; duration=0sec 2024-11-19T05:24:52,568 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:52,568 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:52,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-19T05:24:52,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:52,584 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:24:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194621c2bdf39c49ec9687e243243b4196_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993891548/Put/seqid=0 2024-11-19T05:24:52,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742016_1192 (size=12304) 2024-11-19T05:24:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:52,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:52,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993952700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993952701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993952701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993952702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993952807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993952807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993952818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993952818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:52,911 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/add6e416ef6344f18e5ee82412c2cbeb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb 2024-11-19T05:24:52,918 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into add6e416ef6344f18e5ee82412c2cbeb(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:52,918 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:52,918 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993892433; duration=0sec 2024-11-19T05:24:52,919 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:52,919 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:53,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:53,011 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194621c2bdf39c49ec9687e243243b4196_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194621c2bdf39c49ec9687e243243b4196_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:53,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a3b8bb60caed40c1bba715916c39ad1b, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:53,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a3b8bb60caed40c1bba715916c39ad1b is 175, key is test_row_0/A:col10/1731993891548/Put/seqid=0 2024-11-19T05:24:53,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993953017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993953018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742017_1193 (size=31105) 2024-11-19T05:24:53,028 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a3b8bb60caed40c1bba715916c39ad1b 2024-11-19T05:24:53,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993953028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993953028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cbdaa884729948569b4bc9fd7c45c9b6 is 50, key is test_row_0/B:col10/1731993891548/Put/seqid=0 2024-11-19T05:24:53,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742018_1194 (size=12151) 2024-11-19T05:24:53,049 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cbdaa884729948569b4bc9fd7c45c9b6 2024-11-19T05:24:53,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3ebe30612cc842b09ce56253c5a4c379 is 50, key is test_row_0/C:col10/1731993891548/Put/seqid=0 2024-11-19T05:24:53,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742019_1195 (size=12151) 2024-11-19T05:24:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993953327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993953327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993953337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993953337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,474 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3ebe30612cc842b09ce56253c5a4c379 2024-11-19T05:24:53,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/a3b8bb60caed40c1bba715916c39ad1b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b 2024-11-19T05:24:53,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b, entries=150, sequenceid=253, filesize=30.4 K 2024-11-19T05:24:53,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/cbdaa884729948569b4bc9fd7c45c9b6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6 2024-11-19T05:24:53,489 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T05:24:53,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/3ebe30612cc842b09ce56253c5a4c379 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379 2024-11-19T05:24:53,494 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379, entries=150, sequenceid=253, filesize=11.9 K 2024-11-19T05:24:53,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for a0658358bd5bb50671391f76ab5b1db1 in 911ms, sequenceid=253, compaction requested=false 2024-11-19T05:24:53,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:53,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:53,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-19T05:24:53,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-19T05:24:53,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-19T05:24:53,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-19T05:24:53,501 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.8630 sec 2024-11-19T05:24:53,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-19T05:24:53,818 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:53,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:53,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-19T05:24:53,820 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:53,821 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:53,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:53,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:53,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:24:53,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9725700783e4713a0842cc31b0eb238_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:53,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742020_1196 (size=14944) 2024-11-19T05:24:53,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:53,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9725700783e4713a0842cc31b0eb238_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9725700783e4713a0842cc31b0eb238_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:53,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6358904938504c468ef18f151b407544, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:53,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6358904938504c468ef18f151b407544 is 175, key is test_row_0/A:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:53,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993953873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993953874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993953876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993953877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742021_1197 (size=39899) 2024-11-19T05:24:53,886 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6358904938504c468ef18f151b407544 2024-11-19T05:24:53,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dc70086736ab456eba43dc24fe2eaa71 is 50, key is test_row_0/B:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:53,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742022_1198 (size=12251) 2024-11-19T05:24:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:53,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T05:24:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:53,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:53,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:53,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993953987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993953987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993953987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993953987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T05:24:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993954197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993954197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993954197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993954197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T05:24:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:54,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dc70086736ab456eba43dc24fe2eaa71 2024-11-19T05:24:54,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/bb359ed9e6e54c4495f7bd96fe4be8c5 is 50, key is test_row_0/C:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:54,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742023_1199 (size=12251) 2024-11-19T05:24:54,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/bb359ed9e6e54c4495f7bd96fe4be8c5 2024-11-19T05:24:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6358904938504c468ef18f151b407544 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544 2024-11-19T05:24:54,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544, entries=200, sequenceid=267, filesize=39.0 K 2024-11-19T05:24:54,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dc70086736ab456eba43dc24fe2eaa71 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71 2024-11-19T05:24:54,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T05:24:54,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/bb359ed9e6e54c4495f7bd96fe4be8c5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5 2024-11-19T05:24:54,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5, entries=150, sequenceid=267, filesize=12.0 K 2024-11-19T05:24:54,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for a0658358bd5bb50671391f76ab5b1db1 in 517ms, sequenceid=267, compaction requested=true 2024-11-19T05:24:54,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:54,356 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:54,356 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:54,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:54,357 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:54,357 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:54,357 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,357 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f3c024777dc549469bf9fd09810e8b06, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.2 K 2024-11-19T05:24:54,358 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:54,358 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3c024777dc549469bf9fd09810e8b06, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891486 2024-11-19T05:24:54,358 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:54,358 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,359 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=100.2 K 2024-11-19T05:24:54,359 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544] 2024-11-19T05:24:54,359 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbdaa884729948569b4bc9fd7c45c9b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731993891525 2024-11-19T05:24:54,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting add6e416ef6344f18e5ee82412c2cbeb, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891486 2024-11-19T05:24:54,359 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc70086736ab456eba43dc24fe2eaa71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:54,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b8bb60caed40c1bba715916c39ad1b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731993891525 2024-11-19T05:24:54,360 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6358904938504c468ef18f151b407544, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:54,369 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:54,370 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a96dce795ebb4d28a5e03e23459882fe is 50, key is test_row_0/B:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:54,371 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:54,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fca98349301a4dd7be24a239a7b5ef31_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:54,386 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fca98349301a4dd7be24a239a7b5ef31_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:54,386 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fca98349301a4dd7be24a239a7b5ef31_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:54,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742024_1200 (size=12899) 2024-11-19T05:24:54,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742025_1201 (size=4469) 2024-11-19T05:24:54,396 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#166 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:54,396 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a96dce795ebb4d28a5e03e23459882fe as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a96dce795ebb4d28a5e03e23459882fe 2024-11-19T05:24:54,396 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/97f9db7f096746c2809253fbcc5f5a6e is 175, key is test_row_0/A:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742026_1202 (size=31853) 2024-11-19T05:24:54,404 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into a96dce795ebb4d28a5e03e23459882fe(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:54,404 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:54,404 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993894356; duration=0sec 2024-11-19T05:24:54,405 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:54,405 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:54,405 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:54,406 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:54,406 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:54,406 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,407 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de039aca0f07452e816df60d6917b25e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.2 K 2024-11-19T05:24:54,407 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting de039aca0f07452e816df60d6917b25e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731993891486 2024-11-19T05:24:54,408 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ebe30612cc842b09ce56253c5a4c379, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731993891525 2024-11-19T05:24:54,408 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb359ed9e6e54c4495f7bd96fe4be8c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:54,413 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/97f9db7f096746c2809253fbcc5f5a6e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e 2024-11-19T05:24:54,421 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into 97f9db7f096746c2809253fbcc5f5a6e(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:54,421 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:54,421 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993894356; duration=0sec 2024-11-19T05:24:54,421 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:54,421 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:54,423 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#167 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:54,423 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/1b47c3086660401582ee8d8927dae9ab is 50, key is test_row_0/C:col10/1731993892699/Put/seqid=0 2024-11-19T05:24:54,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742027_1203 (size=12899) 2024-11-19T05:24:54,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-19T05:24:54,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,433 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:24:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111902dacc19a03f411fa27d325da76eaf2c_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993893874/Put/seqid=0 2024-11-19T05:24:54,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:54,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742028_1204 (size=12454) 2024-11-19T05:24:54,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:54,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:54,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993954513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993954515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993954518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993954519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993954627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993954627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993954628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993954627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993954837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993954837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993954838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993954838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:54,842 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/1b47c3086660401582ee8d8927dae9ab as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1b47c3086660401582ee8d8927dae9ab 2024-11-19T05:24:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:54,854 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111902dacc19a03f411fa27d325da76eaf2c_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111902dacc19a03f411fa27d325da76eaf2c_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/005dc71275b34d4dadc9ac5c54ab5152, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:54,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/005dc71275b34d4dadc9ac5c54ab5152 is 175, key is test_row_0/A:col10/1731993893874/Put/seqid=0 2024-11-19T05:24:54,857 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into 1b47c3086660401582ee8d8927dae9ab(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:54,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:54,857 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993894356; duration=0sec 2024-11-19T05:24:54,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:54,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:54,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742029_1205 (size=31255) 2024-11-19T05:24:54,880 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/005dc71275b34d4dadc9ac5c54ab5152 2024-11-19T05:24:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/c5eb1d3f90a246c7841aa9611f53ed03 is 50, key is test_row_0/B:col10/1731993893874/Put/seqid=0 2024-11-19T05:24:54,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742030_1206 (size=12301) 2024-11-19T05:24:54,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/c5eb1d3f90a246c7841aa9611f53ed03 2024-11-19T05:24:54,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/de4866460c8a47f79ca5a02fa7aa9f3b is 50, key is test_row_0/C:col10/1731993893874/Put/seqid=0 2024-11-19T05:24:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742031_1207 (size=12301) 2024-11-19T05:24:54,957 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/de4866460c8a47f79ca5a02fa7aa9f3b 2024-11-19T05:24:54,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:54,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/005dc71275b34d4dadc9ac5c54ab5152 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152 2024-11-19T05:24:54,970 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152, entries=150, sequenceid=292, filesize=30.5 K 2024-11-19T05:24:54,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/c5eb1d3f90a246c7841aa9611f53ed03 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03 2024-11-19T05:24:54,975 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T05:24:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/de4866460c8a47f79ca5a02fa7aa9f3b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b 2024-11-19T05:24:54,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b, entries=150, sequenceid=292, filesize=12.0 K 2024-11-19T05:24:54,984 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for a0658358bd5bb50671391f76ab5b1db1 in 551ms, sequenceid=292, compaction requested=false 2024-11-19T05:24:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-19T05:24:54,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-19T05:24:54,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-19T05:24:54,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1700 sec 2024-11-19T05:24:54,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1640 sec 2024-11-19T05:24:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:55,148 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:55,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:24:55,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195dc3e94223d246b88bf76f4bdaf406b5_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:55,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742032_1208 (size=20074) 2024-11-19T05:24:55,169 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:55,175 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195dc3e94223d246b88bf76f4bdaf406b5_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc3e94223d246b88bf76f4bdaf406b5_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:55,178 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ed193438eb7a48ee80cd8532c32cfb96, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:55,179 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ed193438eb7a48ee80cd8532c32cfb96 is 175, key is test_row_0/A:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:55,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742033_1209 (size=57333) 2024-11-19T05:24:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993955206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993955206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993955206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993955207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993955316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993955317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993955317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993955317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993955527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993955527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993955528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993955528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,590 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ed193438eb7a48ee80cd8532c32cfb96 2024-11-19T05:24:55,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/77c0adf2360b4cf0a4a990b03d2befec is 50, key is test_row_0/B:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:55,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742034_1210 (size=12301) 2024-11-19T05:24:55,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993955837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993955837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993955837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993955837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:55,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-19T05:24:55,977 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:55,977 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-19T05:24:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:55,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:55,979 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:55,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:56,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/77c0adf2360b4cf0a4a990b03d2befec 2024-11-19T05:24:56,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/36691926fde74118b61b6d5940ff5e03 is 50, key is test_row_0/C:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:56,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742035_1211 (size=12301) 2024-11-19T05:24:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:56,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T05:24:56,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:56,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T05:24:56,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:56,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:56,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:56,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:56,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993956347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:56,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:56,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/36691926fde74118b61b6d5940ff5e03 2024-11-19T05:24:56,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/ed193438eb7a48ee80cd8532c32cfb96 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96 2024-11-19T05:24:56,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96, entries=300, sequenceid=309, filesize=56.0 K 2024-11-19T05:24:56,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/77c0adf2360b4cf0a4a990b03d2befec as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec 2024-11-19T05:24:56,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T05:24:56,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/36691926fde74118b61b6d5940ff5e03 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03 2024-11-19T05:24:56,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T05:24:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:56,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T05:24:56,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for a0658358bd5bb50671391f76ab5b1db1 in 1293ms, sequenceid=309, compaction requested=true 2024-11-19T05:24:56,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:56,442 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:56,442 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:56,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120441 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:56,443 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,443 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,443 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a96dce795ebb4d28a5e03e23459882fe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.6 K 2024-11-19T05:24:56,443 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=117.6 K 2024-11-19T05:24:56,443 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96] 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97f9db7f096746c2809253fbcc5f5a6e, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:56,443 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a96dce795ebb4d28a5e03e23459882fe, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:56,444 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 005dc71275b34d4dadc9ac5c54ab5152, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731993893872 2024-11-19T05:24:56,444 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c5eb1d3f90a246c7841aa9611f53ed03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731993893872 2024-11-19T05:24:56,445 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c0adf2360b4cf0a4a990b03d2befec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:56,445 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed193438eb7a48ee80cd8532c32cfb96, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:56,454 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:56,454 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/7fe3ed7debfc4b00b07bd13b02bf9d84 is 50, key is test_row_0/B:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:56,455 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:56,459 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111948b2a88978a749d9bd5499f1e33893cb_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:56,461 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111948b2a88978a749d9bd5499f1e33893cb_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:56,461 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111948b2a88978a749d9bd5499f1e33893cb_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742036_1212 (size=13051) 2024-11-19T05:24:56,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742037_1213 (size=4469) 2024-11-19T05:24:56,479 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#175 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:56,479 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0690740e202e44e187615473b776cee0 is 175, key is test_row_0/A:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:56,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742038_1214 (size=32005) 2024-11-19T05:24:56,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-19T05:24:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,606 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:24:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:56,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196d004bcda931499eb1e1b34237e6d17e_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993895206/Put/seqid=0 2024-11-19T05:24:56,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742039_1215 (size=12454) 2024-11-19T05:24:56,915 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/0690740e202e44e187615473b776cee0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0 2024-11-19T05:24:56,915 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/7fe3ed7debfc4b00b07bd13b02bf9d84 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/7fe3ed7debfc4b00b07bd13b02bf9d84 2024-11-19T05:24:56,921 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into 7fe3ed7debfc4b00b07bd13b02bf9d84(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:56,921 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into 0690740e202e44e187615473b776cee0(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:56,921 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993896442; duration=0sec 2024-11-19T05:24:56,921 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993896442; duration=0sec 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:56,921 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:56,922 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:24:56,922 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:56,922 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:56,922 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:56,922 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1b47c3086660401582ee8d8927dae9ab, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.6 K 2024-11-19T05:24:56,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b47c3086660401582ee8d8927dae9ab, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731993892688 2024-11-19T05:24:56,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting de4866460c8a47f79ca5a02fa7aa9f3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731993893872 2024-11-19T05:24:56,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 36691926fde74118b61b6d5940ff5e03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:56,935 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:56,935 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/21d4acf2f45041029eef5a72e6278907 is 50, key is test_row_0/C:col10/1731993895147/Put/seqid=0 2024-11-19T05:24:56,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742040_1216 (size=13051) 2024-11-19T05:24:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:57,024 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196d004bcda931499eb1e1b34237e6d17e_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d004bcda931499eb1e1b34237e6d17e_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/54c79a211b3b4653b18c2444882492a7, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/54c79a211b3b4653b18c2444882492a7 is 175, key is test_row_0/A:col10/1731993895206/Put/seqid=0 2024-11-19T05:24:57,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742041_1217 (size=31255) 2024-11-19T05:24:57,043 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/54c79a211b3b4653b18c2444882492a7 2024-11-19T05:24:57,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a478c1462e724940bd661e0f15fefe18 is 50, key is test_row_0/B:col10/1731993895206/Put/seqid=0 2024-11-19T05:24:57,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742042_1218 (size=12301) 2024-11-19T05:24:57,085 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a478c1462e724940bd661e0f15fefe18 2024-11-19T05:24:57,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/78aa01636aa44393b5a9c7725ab31ad4 is 50, key is test_row_0/C:col10/1731993895206/Put/seqid=0 2024-11-19T05:24:57,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742043_1219 (size=12301) 2024-11-19T05:24:57,115 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:24:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:57,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993957283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53828 deadline: 1731993957358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,358 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,369 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/21d4acf2f45041029eef5a72e6278907 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/21d4acf2f45041029eef5a72e6278907 2024-11-19T05:24:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53792 deadline: 1731993957367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53776 deadline: 1731993957367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53818 deadline: 1731993957367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,369 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,369 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,371 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in a0658358bd5bb50671391f76ab5b1db1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,375 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into 21d4acf2f45041029eef5a72e6278907(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:57,375 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:57,375 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993896442; duration=0sec 2024-11-19T05:24:57,375 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:57,375 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:24:57,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993957397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/78aa01636aa44393b5a9c7725ab31ad4 2024-11-19T05:24:57,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/54c79a211b3b4653b18c2444882492a7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7 2024-11-19T05:24:57,516 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7, entries=150, sequenceid=329, filesize=30.5 K 2024-11-19T05:24:57,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/a478c1462e724940bd661e0f15fefe18 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18 2024-11-19T05:24:57,520 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T05:24:57,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/78aa01636aa44393b5a9c7725ab31ad4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4 2024-11-19T05:24:57,526 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4, entries=150, sequenceid=329, filesize=12.0 K 2024-11-19T05:24:57,527 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a0658358bd5bb50671391f76ab5b1db1 in 921ms, sequenceid=329, compaction requested=false 2024-11-19T05:24:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-19T05:24:57,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-19T05:24:57,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-19T05:24:57,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-19T05:24:57,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5540 sec 2024-11-19T05:24:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:57,609 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:24:57,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T05:24:57,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d5700013a1124b979caa9922b3cf1336_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:57,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742044_1220 (size=14994) 2024-11-19T05:24:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993957655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993957767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:57,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993957977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:58,020 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:58,024 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119d5700013a1124b979caa9922b3cf1336_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5700013a1124b979caa9922b3cf1336_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:58,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/4fff55fef8a44fa389f18c4ab8cd617f, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:58,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/4fff55fef8a44fa389f18c4ab8cd617f is 175, key is test_row_0/A:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:58,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742045_1221 (size=39949) 2024-11-19T05:24:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-19T05:24:58,127 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:24:58,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:24:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-19T05:24:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:24:58,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:24:58,130 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:24:58,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:24:58,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:24:58,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993958287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:58,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:58,430 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/4fff55fef8a44fa389f18c4ab8cd617f 2024-11-19T05:24:58,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/2dcec5a2f16d471699704e2aa4e99a5a is 50, key is test_row_0/B:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:58,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742046_1222 (size=12301) 2024-11-19T05:24:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:24:58,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:58,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:24:58,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:24:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53806 deadline: 1731993958797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:24:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:24:58,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:24:58,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/2dcec5a2f16d471699704e2aa4e99a5a 2024-11-19T05:24:58,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/808ac6c267b841218e164dea2b50d7d5 is 50, key is test_row_0/C:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:58,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742047_1223 (size=12301) 2024-11-19T05:24:58,858 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-19T05:24:58,858 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:58,858 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,859 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:58,860 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-19T05:24:58,860 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:58,860 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,860 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,860 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:58,861 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-19T05:24:58,861 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:58,861 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,862 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:58,868 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-19T05:24:58,868 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:58,868 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,868 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:58,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:58,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:58,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:24:59,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/808ac6c267b841218e164dea2b50d7d5 2024-11-19T05:24:59,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/4fff55fef8a44fa389f18c4ab8cd617f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f 2024-11-19T05:24:59,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f, entries=200, sequenceid=349, filesize=39.0 K 2024-11-19T05:24:59,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/2dcec5a2f16d471699704e2aa4e99a5a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a 2024-11-19T05:24:59,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-19T05:24:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:24:59,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/808ac6c267b841218e164dea2b50d7d5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5 2024-11-19T05:24:59,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5, entries=150, sequenceid=349, filesize=12.0 K 2024-11-19T05:24:59,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for a0658358bd5bb50671391f76ab5b1db1 in 1663ms, sequenceid=349, compaction requested=true 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:24:59,273 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a0658358bd5bb50671391f76ab5b1db1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:24:59,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:24:59,273 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:59,274 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:59,274 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:59,274 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/B is initiating minor compaction (all files) 2024-11-19T05:24:59,274 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/A is initiating minor compaction (all files) 2024-11-19T05:24:59,274 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/A in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,274 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/B in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,274 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/7fe3ed7debfc4b00b07bd13b02bf9d84, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.8 K 2024-11-19T05:24:59,274 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=100.8 K 2024-11-19T05:24:59,274 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f] 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe3ed7debfc4b00b07bd13b02bf9d84, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0690740e202e44e187615473b776cee0, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a478c1462e724940bd661e0f15fefe18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731993895174 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c79a211b3b4653b18c2444882492a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731993895174 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dcec5a2f16d471699704e2aa4e99a5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731993897278 2024-11-19T05:24:59,275 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fff55fef8a44fa389f18c4ab8cd617f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731993897269 2024-11-19T05:24:59,284 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:59,285 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/e32f18dd68e940bd9a38e44c26fe59e3 is 50, key is test_row_0/B:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:59,287 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:59,295 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a6f3217bc6e84f4e8d5c7e894214ae3b_a0658358bd5bb50671391f76ab5b1db1 store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742048_1224 (size=13153) 2024-11-19T05:24:59,323 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a6f3217bc6e84f4e8d5c7e894214ae3b_a0658358bd5bb50671391f76ab5b1db1, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:59,323 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a6f3217bc6e84f4e8d5c7e894214ae3b_a0658358bd5bb50671391f76ab5b1db1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:59,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742049_1225 (size=4469) 2024-11-19T05:24:59,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-19T05:24:59,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,357 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T05:24:59,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:59,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. as already flushing 2024-11-19T05:24:59,371 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-19T05:24:59,371 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:59,371 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111980247e2696124086ae9699faf3572952_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993897641/Put/seqid=0 2024-11-19T05:24:59,371 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:59,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742050_1226 (size=12454) 2024-11-19T05:24:59,390 INFO [Thread-621 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-621. 2024-11-19T05:24:59,390 DEBUG [Thread-621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:59,390 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,391 DEBUG [Thread-621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:59,402 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-19T05:24:59,402 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-19T05:24:59,402 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:59,402 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,402 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:59,402 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,402 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,402 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,402 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:59,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:24:59,704 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/e32f18dd68e940bd9a38e44c26fe59e3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/e32f18dd68e940bd9a38e44c26fe59e3 2024-11-19T05:24:59,710 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/B of a0658358bd5bb50671391f76ab5b1db1 into e32f18dd68e940bd9a38e44c26fe59e3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:24:59,710 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:24:59,710 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/B, priority=13, startTime=1731993899273; duration=0sec 2024-11-19T05:24:59,711 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:24:59,711 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:B 2024-11-19T05:24:59,711 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:24:59,712 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:24:59,712 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): a0658358bd5bb50671391f76ab5b1db1/C is initiating minor compaction (all files) 2024-11-19T05:24:59,712 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a0658358bd5bb50671391f76ab5b1db1/C in TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:24:59,713 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/21d4acf2f45041029eef5a72e6278907, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp, totalSize=36.8 K 2024-11-19T05:24:59,713 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 21d4acf2f45041029eef5a72e6278907, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993894512 2024-11-19T05:24:59,714 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 78aa01636aa44393b5a9c7725ab31ad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731993895174 2024-11-19T05:24:59,714 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 808ac6c267b841218e164dea2b50d7d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731993897278 2024-11-19T05:24:59,722 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:59,723 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/6566d9580eea436aa2987f760b642a84 is 50, key is test_row_0/C:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:59,728 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a0658358bd5bb50671391f76ab5b1db1#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:24:59,729 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6e1e3ea06edd4621889e64127ab364b8 is 175, key is test_row_0/A:col10/1731993897281/Put/seqid=0 2024-11-19T05:24:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742051_1227 (size=13153) 2024-11-19T05:24:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742052_1228 (size=32107) 2024-11-19T05:24:59,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:24:59,779 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111980247e2696124086ae9699faf3572952_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111980247e2696124086ae9699faf3572952_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:24:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86ef4ae42fa14b1c97c5b5400f1b3811, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:24:59,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86ef4ae42fa14b1c97c5b5400f1b3811 is 175, key is test_row_0/A:col10/1731993897641/Put/seqid=0 2024-11-19T05:24:59,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742053_1229 (size=31255) 2024-11-19T05:24:59,818 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-19T05:24:59,818 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:24:59,818 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,818 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:24:59,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:00,147 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/6e1e3ea06edd4621889e64127ab364b8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6e1e3ea06edd4621889e64127ab364b8 2024-11-19T05:25:00,147 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/6566d9580eea436aa2987f760b642a84 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/6566d9580eea436aa2987f760b642a84 2024-11-19T05:25:00,152 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/C of a0658358bd5bb50671391f76ab5b1db1 into 6566d9580eea436aa2987f760b642a84(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:00,152 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a0658358bd5bb50671391f76ab5b1db1/A of a0658358bd5bb50671391f76ab5b1db1 into 6e1e3ea06edd4621889e64127ab364b8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:00,152 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:25:00,152 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:25:00,152 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/A, priority=13, startTime=1731993899273; duration=0sec 2024-11-19T05:25:00,152 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., storeName=a0658358bd5bb50671391f76ab5b1db1/C, priority=13, startTime=1731993899273; duration=0sec 2024-11-19T05:25:00,153 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:00,153 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:A 2024-11-19T05:25:00,153 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:00,153 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a0658358bd5bb50671391f76ab5b1db1:C 2024-11-19T05:25:00,190 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86ef4ae42fa14b1c97c5b5400f1b3811 2024-11-19T05:25:00,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dcdea3323e58455ead0e4463f9c4e1cc is 50, key is test_row_0/B:col10/1731993897641/Put/seqid=0 2024-11-19T05:25:00,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742054_1230 (size=12301) 2024-11-19T05:25:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:25:00,605 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dcdea3323e58455ead0e4463f9c4e1cc 2024-11-19T05:25:00,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/69c168df09ab4c5b9e646709faa53c67 is 50, key is test_row_0/C:col10/1731993897641/Put/seqid=0 2024-11-19T05:25:00,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742055_1231 (size=12301) 2024-11-19T05:25:01,023 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/69c168df09ab4c5b9e646709faa53c67 2024-11-19T05:25:01,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/86ef4ae42fa14b1c97c5b5400f1b3811 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86ef4ae42fa14b1c97c5b5400f1b3811 2024-11-19T05:25:01,040 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86ef4ae42fa14b1c97c5b5400f1b3811, entries=150, sequenceid=368, filesize=30.5 K 2024-11-19T05:25:01,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/dcdea3323e58455ead0e4463f9c4e1cc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dcdea3323e58455ead0e4463f9c4e1cc 2024-11-19T05:25:01,046 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dcdea3323e58455ead0e4463f9c4e1cc, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T05:25:01,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/69c168df09ab4c5b9e646709faa53c67 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/69c168df09ab4c5b9e646709faa53c67 2024-11-19T05:25:01,052 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/69c168df09ab4c5b9e646709faa53c67, entries=150, sequenceid=368, filesize=12.0 K 2024-11-19T05:25:01,053 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=33.54 KB/34350 for a0658358bd5bb50671391f76ab5b1db1 in 1696ms, sequenceid=368, compaction requested=false 2024-11-19T05:25:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a0658358bd5bb50671391f76ab5b1db1: 2024-11-19T05:25:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:25:01,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-19T05:25:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-19T05:25:01,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-19T05:25:01,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-19T05:25:01,057 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.9280 sec 2024-11-19T05:25:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-19T05:25:02,308 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3512 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3634 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1545 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4632 rows 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1540 2024-11-19T05:25:02,308 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4620 rows 2024-11-19T05:25:02,308 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:25:02,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:25:02,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:02,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:02,309 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:02,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:25:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:02,314 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993902314"}]},"ts":"1731993902314"} 2024-11-19T05:25:02,316 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:25:02,316 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:25:02,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:25:02,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, UNASSIGN}] 2024-11-19T05:25:02,320 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, UNASSIGN 2024-11-19T05:25:02,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:02,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, UNASSIGN because future has completed 2024-11-19T05:25:02,323 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:25:02,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:02,475 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:02,475 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:25:02,475 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing a0658358bd5bb50671391f76ab5b1db1, disabling compactions & flushes 2024-11-19T05:25:02,475 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:25:02,475 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:25:02,475 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. after waiting 0 ms 2024-11-19T05:25:02,475 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:25:02,475 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing a0658358bd5bb50671391f76ab5b1db1 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T05:25:02,481 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197cb5c327268343ecbea09d021122f027_a0658358bd5bb50671391f76ab5b1db1 is 50, key is test_row_0/A:col10/1731993899817/Put/seqid=0 2024-11-19T05:25:02,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742056_1232 (size=12454) 2024-11-19T05:25:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:02,886 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:02,890 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197cb5c327268343ecbea09d021122f027_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb5c327268343ecbea09d021122f027_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:02,891 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/56e9f3f911374eb29885534f3e169cec, store: [table=TestAcidGuarantees family=A region=a0658358bd5bb50671391f76ab5b1db1] 2024-11-19T05:25:02,892 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/56e9f3f911374eb29885534f3e169cec is 175, key is test_row_0/A:col10/1731993899817/Put/seqid=0 2024-11-19T05:25:02,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742057_1233 (size=31255) 2024-11-19T05:25:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:03,296 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/56e9f3f911374eb29885534f3e169cec 2024-11-19T05:25:03,304 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/565fb53494e241c8892d2f0c4bc54f4f is 50, key is test_row_0/B:col10/1731993899817/Put/seqid=0 2024-11-19T05:25:03,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742058_1234 (size=12301) 2024-11-19T05:25:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:03,723 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/565fb53494e241c8892d2f0c4bc54f4f 2024-11-19T05:25:03,729 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/321bd0dfb0464694938cc07c95b9f8ea is 50, key is test_row_0/C:col10/1731993899817/Put/seqid=0 2024-11-19T05:25:03,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742059_1235 (size=12301) 2024-11-19T05:25:04,133 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/321bd0dfb0464694938cc07c95b9f8ea 2024-11-19T05:25:04,137 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/A/56e9f3f911374eb29885534f3e169cec as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/56e9f3f911374eb29885534f3e169cec 2024-11-19T05:25:04,141 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/56e9f3f911374eb29885534f3e169cec, entries=150, sequenceid=379, filesize=30.5 K 2024-11-19T05:25:04,141 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/B/565fb53494e241c8892d2f0c4bc54f4f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/565fb53494e241c8892d2f0c4bc54f4f 2024-11-19T05:25:04,145 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/565fb53494e241c8892d2f0c4bc54f4f, entries=150, sequenceid=379, filesize=12.0 K 2024-11-19T05:25:04,145 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/.tmp/C/321bd0dfb0464694938cc07c95b9f8ea as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/321bd0dfb0464694938cc07c95b9f8ea 2024-11-19T05:25:04,149 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/321bd0dfb0464694938cc07c95b9f8ea, entries=150, sequenceid=379, filesize=12.0 K 2024-11-19T05:25:04,150 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a0658358bd5bb50671391f76ab5b1db1 in 1674ms, sequenceid=379, compaction requested=true 2024-11-19T05:25:04,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f] to archive 2024-11-19T05:25:04,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:04,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a9d93019f98c4149aa0d1298edf309d4 2024-11-19T05:25:04,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0ac84bf42e7f4ec4b4ad2ed6ac276619 2024-11-19T05:25:04,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/bf5f904998884f5680a8ae0d44cd0c66 2024-11-19T05:25:04,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/c4277760a2c147b1ab134091d77728ce 2024-11-19T05:25:04,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/b6ec9ccc0b6f4409ab4de1d056d89d2d 2024-11-19T05:25:04,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/98d73090acd144708b1bcc6812b18dbc 2024-11-19T05:25:04,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/2bfbd8dd11cb4cfa9e7d8b4a1a366d08 2024-11-19T05:25:04,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/aee71dba586b4facacf86c17fa44d2ac 2024-11-19T05:25:04,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/598a022a3a2f4cafaa91c7b933a32fc4 2024-11-19T05:25:04,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ee68cbfe01e7481cac2ac39b3476d398 2024-11-19T05:25:04,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86691bbd12eb4a9495dd18ce9372fd82 2024-11-19T05:25:04,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/cd347a20a5974260bb8b4964f435402c 2024-11-19T05:25:04,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/77715c06466c4b7488c181197f2bfabe 2024-11-19T05:25:04,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/8cfe460eca8d4ef191389f313813fb83 2024-11-19T05:25:04,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/5347ca807a9c40aab4c70eb59932024c 2024-11-19T05:25:04,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/e74983abb7f641389187fd666ce9b4fc 2024-11-19T05:25:04,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/add6e416ef6344f18e5ee82412c2cbeb 2024-11-19T05:25:04,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/a3b8bb60caed40c1bba715916c39ad1b 2024-11-19T05:25:04,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6358904938504c468ef18f151b407544 2024-11-19T05:25:04,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/97f9db7f096746c2809253fbcc5f5a6e 2024-11-19T05:25:04,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/005dc71275b34d4dadc9ac5c54ab5152 2024-11-19T05:25:04,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/ed193438eb7a48ee80cd8532c32cfb96 2024-11-19T05:25:04,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/0690740e202e44e187615473b776cee0 2024-11-19T05:25:04,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/54c79a211b3b4653b18c2444882492a7 2024-11-19T05:25:04,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/4fff55fef8a44fa389f18c4ab8cd617f 2024-11-19T05:25:04,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/d40a30b094634fa9b4608559e72c04e9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/bf55bad0e82242afa17a547254f477a3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/70387ba6182a490a85e42237a674b4ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/64117a79f91b4ffc9883ff955b096e08, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f3c024777dc549469bf9fd09810e8b06, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a96dce795ebb4d28a5e03e23459882fe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/7fe3ed7debfc4b00b07bd13b02bf9d84, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a] to archive 2024-11-19T05:25:04,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:04,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/15c0089191cc465cb7db18d574f800ce 2024-11-19T05:25:04,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/4e160c4f9b5044e39cd2d6094d082863 2024-11-19T05:25:04,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/d40a30b094634fa9b4608559e72c04e9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/d40a30b094634fa9b4608559e72c04e9 2024-11-19T05:25:04,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/67f82743f43f45f09cbe4acf10dea0f0 2024-11-19T05:25:04,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cda5598ea52140ea8ffc4976ed92f7c8 2024-11-19T05:25:04,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/bf55bad0e82242afa17a547254f477a3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/bf55bad0e82242afa17a547254f477a3 2024-11-19T05:25:04,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4533ba3c3a64615af462533cd7e091c 2024-11-19T05:25:04,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/75d4e04c3ed047259e95462b99cbc3f5 2024-11-19T05:25:04,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/70387ba6182a490a85e42237a674b4ac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/70387ba6182a490a85e42237a674b4ac 2024-11-19T05:25:04,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a062a38fdf8e49319712ad9c8418fb01 2024-11-19T05:25:04,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/09b9a8619d524463a769ffcd829a42ae 2024-11-19T05:25:04,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/49787534725a4539a253832b00e5af71 2024-11-19T05:25:04,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/64117a79f91b4ffc9883ff955b096e08 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/64117a79f91b4ffc9883ff955b096e08 2024-11-19T05:25:04,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f4de5a7532ff415bb230db595cb35286 2024-11-19T05:25:04,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/47ebbeff1ee74bcd9a40fda5d1a4cac1 2024-11-19T05:25:04,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f3c024777dc549469bf9fd09810e8b06 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/f3c024777dc549469bf9fd09810e8b06 2024-11-19T05:25:04,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/b9952174cc0e48138ac52f385a50d345 2024-11-19T05:25:04,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/cbdaa884729948569b4bc9fd7c45c9b6 2024-11-19T05:25:04,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a96dce795ebb4d28a5e03e23459882fe to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a96dce795ebb4d28a5e03e23459882fe 2024-11-19T05:25:04,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dc70086736ab456eba43dc24fe2eaa71 2024-11-19T05:25:04,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/c5eb1d3f90a246c7841aa9611f53ed03 2024-11-19T05:25:04,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/7fe3ed7debfc4b00b07bd13b02bf9d84 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/7fe3ed7debfc4b00b07bd13b02bf9d84 2024-11-19T05:25:04,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/77c0adf2360b4cf0a4a990b03d2befec 2024-11-19T05:25:04,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/a478c1462e724940bd661e0f15fefe18 2024-11-19T05:25:04,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/2dcec5a2f16d471699704e2aa4e99a5a 2024-11-19T05:25:04,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/56a47cb73fb44bafbe03cffffb98e91b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/76bd20fc953e493c8ea3216bcc23c809, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/cb2f73e02a1d40589a6208c8a73bf2fa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e57bb4e310a048d1971e742c8d7afcae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de039aca0f07452e816df60d6917b25e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1b47c3086660401582ee8d8927dae9ab, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/21d4acf2f45041029eef5a72e6278907, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5] to archive 2024-11-19T05:25:04,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:04,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/a1896f0005a645eda94d128011c304de 2024-11-19T05:25:04,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/448ae2e26d6d468eb1285088dd3306a4 2024-11-19T05:25:04,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/56a47cb73fb44bafbe03cffffb98e91b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/56a47cb73fb44bafbe03cffffb98e91b 2024-11-19T05:25:04,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b125aefc46b8434789d481188f247580 2024-11-19T05:25:04,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b25b9e5b89794b19bb94fa14bebe45d2 2024-11-19T05:25:04,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/76bd20fc953e493c8ea3216bcc23c809 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/76bd20fc953e493c8ea3216bcc23c809 2024-11-19T05:25:04,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/7ba114b6b30b4c27a4dcdc2df02318da 2024-11-19T05:25:04,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1bb6a2e5faab4608ae21aacf282af47b 2024-11-19T05:25:04,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/cb2f73e02a1d40589a6208c8a73bf2fa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/cb2f73e02a1d40589a6208c8a73bf2fa 2024-11-19T05:25:04,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3b626bc415ce4ff1b774e59b3f21ac72 2024-11-19T05:25:04,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/b0308c4e30904256a1e747a050314826 2024-11-19T05:25:04,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/262284eb5c444f31904065e1831ddd66 2024-11-19T05:25:04,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e57bb4e310a048d1971e742c8d7afcae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e57bb4e310a048d1971e742c8d7afcae 2024-11-19T05:25:04,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/e1ec65d5d7e44469be485338c9b3fecb 2024-11-19T05:25:04,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/8f419478754747618ed17f3f142d89ba 2024-11-19T05:25:04,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de039aca0f07452e816df60d6917b25e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de039aca0f07452e816df60d6917b25e 2024-11-19T05:25:04,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/485a2c8640d3472fa366cd285155bc51 2024-11-19T05:25:04,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/3ebe30612cc842b09ce56253c5a4c379 2024-11-19T05:25:04,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1b47c3086660401582ee8d8927dae9ab to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/1b47c3086660401582ee8d8927dae9ab 2024-11-19T05:25:04,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/bb359ed9e6e54c4495f7bd96fe4be8c5 2024-11-19T05:25:04,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/de4866460c8a47f79ca5a02fa7aa9f3b 2024-11-19T05:25:04,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/21d4acf2f45041029eef5a72e6278907 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/21d4acf2f45041029eef5a72e6278907 2024-11-19T05:25:04,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/36691926fde74118b61b6d5940ff5e03 2024-11-19T05:25:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/78aa01636aa44393b5a9c7725ab31ad4 2024-11-19T05:25:04,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/808ac6c267b841218e164dea2b50d7d5 2024-11-19T05:25:04,235 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-19T05:25:04,236 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. 2024-11-19T05:25:04,236 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for a0658358bd5bb50671391f76ab5b1db1: Waiting for close lock at 1731993902475Running coprocessor pre-close hooks at 1731993902475Disabling compacts and flushes for region at 1731993902475Disabling writes for close at 1731993902475Obtaining lock to block concurrent updates at 1731993902475Preparing flush snapshotting stores in a0658358bd5bb50671391f76ab5b1db1 at 1731993902475Finished memstore snapshotting TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731993902476 (+1 ms)Flushing stores of TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1. at 1731993902476Flushing a0658358bd5bb50671391f76ab5b1db1/A: creating writer at 1731993902476Flushing mob file a0658358bd5bb50671391f76ab5b1db1/A: appending metadata at 1731993902481 (+5 ms)Flushing mob file a0658358bd5bb50671391f76ab5b1db1/A: closing flushed file at 1731993902481Flushing a0658358bd5bb50671391f76ab5b1db1/A: appending metadata at 1731993902891 (+410 ms)Flushing a0658358bd5bb50671391f76ab5b1db1/A: closing flushed file at 1731993902891Flushing a0658358bd5bb50671391f76ab5b1db1/B: creating writer at 1731993903300 (+409 ms)Flushing a0658358bd5bb50671391f76ab5b1db1/B: appending metadata at 1731993903304 (+4 ms)Flushing a0658358bd5bb50671391f76ab5b1db1/B: closing flushed file at 1731993903304Flushing a0658358bd5bb50671391f76ab5b1db1/C: creating writer at 1731993903727 (+423 ms)Flushing a0658358bd5bb50671391f76ab5b1db1/C: appending metadata at 1731993903729 (+2 ms)Flushing a0658358bd5bb50671391f76ab5b1db1/C: closing flushed file at 1731993903729Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a7f8e5b: reopening flushed file at 1731993904136 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ebe7a1d: reopening flushed file at 1731993904141 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@49904166: reopening flushed file at 1731993904145 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for a0658358bd5bb50671391f76ab5b1db1 in 1674ms, sequenceid=379, compaction requested=true at 1731993904150 (+5 ms)Writing region close event to WAL at 1731993904232 (+82 ms)Running coprocessor post-close hooks at 1731993904235 (+3 ms)Closed at 1731993904236 (+1 ms) 2024-11-19T05:25:04,238 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,238 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=a0658358bd5bb50671391f76ab5b1db1, regionState=CLOSED 2024-11-19T05:25:04,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:04,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-19T05:25:04,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure a0658358bd5bb50671391f76ab5b1db1, server=f990f635ffd0,38449,1731993838792 in 1.9170 sec 2024-11-19T05:25:04,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-19T05:25:04,243 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a0658358bd5bb50671391f76ab5b1db1, UNASSIGN in 1.9230 sec 2024-11-19T05:25:04,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-19T05:25:04,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9260 sec 2024-11-19T05:25:04,246 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993904246"}]},"ts":"1731993904246"} 2024-11-19T05:25:04,247 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:25:04,247 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:25:04,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9370 sec 2024-11-19T05:25:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-19T05:25:04,459 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:04,460 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:25:04,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,466 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T05:25:04,468 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,472 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,477 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits] 2024-11-19T05:25:04,481 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/56e9f3f911374eb29885534f3e169cec to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/56e9f3f911374eb29885534f3e169cec 2024-11-19T05:25:04,483 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6e1e3ea06edd4621889e64127ab364b8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/6e1e3ea06edd4621889e64127ab364b8 2024-11-19T05:25:04,486 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86ef4ae42fa14b1c97c5b5400f1b3811 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/A/86ef4ae42fa14b1c97c5b5400f1b3811 2024-11-19T05:25:04,489 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/565fb53494e241c8892d2f0c4bc54f4f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/565fb53494e241c8892d2f0c4bc54f4f 2024-11-19T05:25:04,491 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dcdea3323e58455ead0e4463f9c4e1cc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/dcdea3323e58455ead0e4463f9c4e1cc 2024-11-19T05:25:04,493 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/e32f18dd68e940bd9a38e44c26fe59e3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/B/e32f18dd68e940bd9a38e44c26fe59e3 2024-11-19T05:25:04,497 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/321bd0dfb0464694938cc07c95b9f8ea to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/321bd0dfb0464694938cc07c95b9f8ea 2024-11-19T05:25:04,499 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/6566d9580eea436aa2987f760b642a84 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/6566d9580eea436aa2987f760b642a84 2024-11-19T05:25:04,502 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/69c168df09ab4c5b9e646709faa53c67 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/C/69c168df09ab4c5b9e646709faa53c67 2024-11-19T05:25:04,505 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits/382.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1/recovered.edits/382.seqid 2024-11-19T05:25:04,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,506 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:25:04,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:25:04,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T05:25:04,511 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111902dacc19a03f411fa27d325da76eaf2c_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111902dacc19a03f411fa27d325da76eaf2c_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,512 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111905aad005c45243a9b302bba4d92d525a_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111905aad005c45243a9b302bba4d92d525a_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,513 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e095605de1b4081a2bd39030c4defe2_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191e095605de1b4081a2bd39030c4defe2_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193968e7057f104ecca86c87922e2bc647_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193968e7057f104ecca86c87922e2bc647_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194621c2bdf39c49ec9687e243243b4196_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194621c2bdf39c49ec9687e243243b4196_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,516 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ca5cbbe4382414184cc8a2360537377_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195ca5cbbe4382414184cc8a2360537377_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,517 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc3e94223d246b88bf76f4bdaf406b5_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195dc3e94223d246b88bf76f4bdaf406b5_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,518 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962bbad573e8e4072a7b738bda90e3f65_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111962bbad573e8e4072a7b738bda90e3f65_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,519 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968998536cff140b28b5fc1d724ac8cfc_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111968998536cff140b28b5fc1d724ac8cfc_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,520 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d004bcda931499eb1e1b34237e6d17e_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196d004bcda931499eb1e1b34237e6d17e_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb5c327268343ecbea09d021122f027_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411197cb5c327268343ecbea09d021122f027_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,521 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111980247e2696124086ae9699faf3572952_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111980247e2696124086ae9699faf3572952_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,522 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8edf41b428840809d0ba66f4af82fd6_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b8edf41b428840809d0ba66f4af82fd6_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,523 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9725700783e4713a0842cc31b0eb238_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9725700783e4713a0842cc31b0eb238_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,524 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9f1e0c2ce574035a3a7e3983c67df62_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9f1e0c2ce574035a3a7e3983c67df62_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,525 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd53804ad1ae46d8b0626ab54b6e3d03_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119cd53804ad1ae46d8b0626ab54b6e3d03_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,526 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5700013a1124b979caa9922b3cf1336_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119d5700013a1124b979caa9922b3cf1336_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,526 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119df2fabc2475a42549374c9cbf84288c4_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119df2fabc2475a42549374c9cbf84288c4_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,527 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef8e04c7197f470f99f3ad116b1ec9e7_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ef8e04c7197f470f99f3ad116b1ec9e7_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,528 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdd17e8adf7149e2b61b02a9427d46ac_a0658358bd5bb50671391f76ab5b1db1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119fdd17e8adf7149e2b61b02a9427d46ac_a0658358bd5bb50671391f76ab5b1db1 2024-11-19T05:25:04,529 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:25:04,531 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,533 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:25:04,535 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:25:04,537 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,537 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:25:04,537 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731993904537"}]},"ts":"9223372036854775807"} 2024-11-19T05:25:04,539 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:25:04,539 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a0658358bd5bb50671391f76ab5b1db1, NAME => 'TestAcidGuarantees,,1731993875671.a0658358bd5bb50671391f76ab5b1db1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:25:04,539 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:25:04,539 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731993904539"}]},"ts":"9223372036854775807"} 2024-11-19T05:25:04,541 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:25:04,542 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-19T05:25:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-19T05:25:04,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:25:04,578 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:04,587 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1179197976_22 at /127.0.0.1:34988 [Waiting for operation #964] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-299407974_22 at /127.0.0.1:34972 [Waiting for operation #959] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1179197976_22 at /127.0.0.1:44448 [Waiting for operation #1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-299407974_22 at /127.0.0.1:44354 [Waiting for operation #1101] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=395 (was 403), ProcessCount=10 (was 11), AvailableMemoryMB=12914 (was 12152) - AvailableMemoryMB LEAK? - 2024-11-19T05:25:04,596 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=395, ProcessCount=9, AvailableMemoryMB=12914 2024-11-19T05:25:04,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:25:04,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:25:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:04,599 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:25:04,599 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:04,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-19T05:25:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:04,600 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:25:04,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742060_1236 (size=959) 2024-11-19T05:25:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:04,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:05,007 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:25:05,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742061_1237 (size=53) 2024-11-19T05:25:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7de0717bd410d1dc7ce54e46b0058ca3, disabling compactions & flushes 2024-11-19T05:25:05,413 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. after waiting 0 ms 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,413 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7de0717bd410d1dc7ce54e46b0058ca3: Waiting for close lock at 1731993905413Disabling compacts and flushes for region at 1731993905413Disabling writes for close at 1731993905413Writing region close event to WAL at 1731993905413Closed at 1731993905413 2024-11-19T05:25:05,415 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:25:05,415 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993905415"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993905415"}]},"ts":"1731993905415"} 2024-11-19T05:25:05,417 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:25:05,418 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:25:05,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993905418"}]},"ts":"1731993905418"} 2024-11-19T05:25:05,420 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:25:05,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, ASSIGN}] 2024-11-19T05:25:05,421 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, ASSIGN 2024-11-19T05:25:05,422 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:25:05,573 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7de0717bd410d1dc7ce54e46b0058ca3, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:05,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, ASSIGN because future has completed 2024-11-19T05:25:05,575 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:05,730 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,730 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:25:05,731 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,731 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:25:05,731 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,731 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,732 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,734 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7de0717bd410d1dc7ce54e46b0058ca3 columnFamilyName A 2024-11-19T05:25:05,734 DEBUG [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:05,734 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(327): Store=7de0717bd410d1dc7ce54e46b0058ca3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:05,734 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,735 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7de0717bd410d1dc7ce54e46b0058ca3 columnFamilyName B 2024-11-19T05:25:05,735 DEBUG [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:05,736 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(327): Store=7de0717bd410d1dc7ce54e46b0058ca3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:05,736 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,737 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7de0717bd410d1dc7ce54e46b0058ca3 columnFamilyName C 2024-11-19T05:25:05,737 DEBUG [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:05,738 INFO [StoreOpener-7de0717bd410d1dc7ce54e46b0058ca3-1 {}] regionserver.HStore(327): Store=7de0717bd410d1dc7ce54e46b0058ca3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:05,738 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,739 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,739 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,740 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,740 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,741 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:25:05,742 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,745 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:25:05,745 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened 7de0717bd410d1dc7ce54e46b0058ca3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60488038, jitterRate=-0.0986579954624176}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:25:05,745 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:05,746 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for 7de0717bd410d1dc7ce54e46b0058ca3: Running coprocessor pre-open hook at 1731993905731Writing region info on filesystem at 1731993905731Initializing all the Stores at 1731993905732 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993905732Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993905732Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993905732Cleaning up temporary data from old regions at 1731993905740 (+8 ms)Running coprocessor post-open hooks at 1731993905745 (+5 ms)Region opened successfully at 1731993905746 (+1 ms) 2024-11-19T05:25:05,747 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., pid=61, masterSystemTime=1731993905727 2024-11-19T05:25:05,749 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,749 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:05,749 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=7de0717bd410d1dc7ce54e46b0058ca3, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:05,751 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:05,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-19T05:25:05,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 in 177 msec 2024-11-19T05:25:05,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-19T05:25:05,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, ASSIGN in 337 msec 2024-11-19T05:25:05,760 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:25:05,760 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993905760"}]},"ts":"1731993905760"} 2024-11-19T05:25:05,762 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:25:05,763 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:25:05,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-19T05:25:06,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-19T05:25:06,757 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:06,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@730f6619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169d11ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,769 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:25:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:25:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@388d9de3 2024-11-19T05:25:06,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:25:06,771 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:25:06,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4399e0a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,774 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52826, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,775 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@133533a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,776 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f81780, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,778 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dc7c855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,780 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,780 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5deabc16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,782 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@360b86a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15911e08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,787 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339cb1bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d9cbce5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,791 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc12f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8815e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d15305c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7430fe7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,797 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd56d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,799 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28398524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,801 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a36865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,802 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70dc88d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,803 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,803 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,803 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3675059a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,805 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e80fb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:06,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:06,807 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f652dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:06,808 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:06,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17514f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:06,809 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,809 DEBUG [Thread-1119 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,809 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,809 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,811 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:06,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,812 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-19T05:25:06,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,814 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,816 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:06,817 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:06,817 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:06,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:06,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:25:06,836 DEBUG [Thread-1123 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/f16f5a95435649b8b2f4b8d177bac660 is 50, key is test_row_0/A:col10/1731993906825/Put/seqid=0 2024-11-19T05:25:06,860 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,862 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,862 DEBUG [Thread-1125 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,862 DEBUG [Thread-1114 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,863 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,864 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,865 DEBUG [Thread-1114 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,865 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,871 DEBUG [Thread-1121 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,877 DEBUG [Thread-1121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:06,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742062_1238 (size=14341) 2024-11-19T05:25:06,878 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:06,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/f16f5a95435649b8b2f4b8d177bac660 2024-11-19T05:25:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:25:06,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993966880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993966882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:06,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:06,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993966882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:06,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:06,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993966885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:06,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993966886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:06,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:06,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:06,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/da448e01c91f4be5bd79448c7632f95c is 50, key is test_row_0/B:col10/1731993906825/Put/seqid=0 2024-11-19T05:25:06,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742063_1239 (size=12001) 2024-11-19T05:25:06,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/da448e01c91f4be5bd79448c7632f95c 2024-11-19T05:25:06,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:06,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:06,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/21605b7694164b08ba99b99c20b3d6d2 is 50, key is test_row_0/C:col10/1731993906825/Put/seqid=0 2024-11-19T05:25:06,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993966997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993966997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993966997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993966997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993966997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742064_1240 (size=12001) 2024-11-19T05:25:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:07,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993967208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993967210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993967210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993967210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993967210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:07,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/21605b7694164b08ba99b99c20b3d6d2 2024-11-19T05:25:07,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/f16f5a95435649b8b2f4b8d177bac660 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660 2024-11-19T05:25:07,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660, entries=200, sequenceid=12, filesize=14.0 K 2024-11-19T05:25:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/da448e01c91f4be5bd79448c7632f95c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c 2024-11-19T05:25:07,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T05:25:07,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/21605b7694164b08ba99b99c20b3d6d2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2 2024-11-19T05:25:07,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,504 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2, entries=150, sequenceid=12, filesize=11.7 K 2024-11-19T05:25:07,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 7de0717bd410d1dc7ce54e46b0058ca3 in 676ms, sequenceid=12, compaction requested=false 2024-11-19T05:25:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:07,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:07,520 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:07,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T05:25:07,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/89fc1de128c74d598dfe3010dc8506b9 is 50, key is test_row_0/A:col10/1731993907520/Put/seqid=0 2024-11-19T05:25:07,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993967524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993967525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993967529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993967532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993967532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742065_1241 (size=12001) 2024-11-19T05:25:07,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/89fc1de128c74d598dfe3010dc8506b9 2024-11-19T05:25:07,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/17c3109d0f654b2ea58d34b9d92232ad is 50, key is test_row_0/B:col10/1731993907520/Put/seqid=0 2024-11-19T05:25:07,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742066_1242 (size=12001) 2024-11-19T05:25:07,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/17c3109d0f654b2ea58d34b9d92232ad 2024-11-19T05:25:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993967637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993967637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/62276e49c2874e70a21f253c1d01bcbc is 50, key is test_row_0/C:col10/1731993907520/Put/seqid=0 2024-11-19T05:25:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993967648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993967648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993967648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742067_1243 (size=12001) 2024-11-19T05:25:07,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993967847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993967847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993967857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993967858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:07,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993967858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:07,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:07,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:07,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:07,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/62276e49c2874e70a21f253c1d01bcbc 2024-11-19T05:25:08,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/89fc1de128c74d598dfe3010dc8506b9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9 2024-11-19T05:25:08,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:25:08,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/17c3109d0f654b2ea58d34b9d92232ad as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad 2024-11-19T05:25:08,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:25:08,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/62276e49c2874e70a21f253c1d01bcbc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc 2024-11-19T05:25:08,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:25:08,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 7de0717bd410d1dc7ce54e46b0058ca3 in 602ms, sequenceid=41, compaction requested=false 2024-11-19T05:25:08,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:08,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:08,162 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:08,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:25:08,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/bb9e7f968b1941f2ab1d28b5741cfb75 is 50, key is test_row_0/A:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:08,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993968193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993968194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993968195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993968196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742068_1244 (size=14341) 2024-11-19T05:25:08,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993968201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993968307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993968307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993968307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993968307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993968317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993968518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993968518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993968518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993968517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993968527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/bb9e7f968b1941f2ab1d28b5741cfb75 2024-11-19T05:25:08,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5c82b8de53124db8879031d2d44eb9f1 is 50, key is test_row_0/B:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:08,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742069_1245 (size=12001) 2024-11-19T05:25:08,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993968827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993968827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993968827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993968828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993968837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:08,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:08,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:08,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:08,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:09,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5c82b8de53124db8879031d2d44eb9f1 2024-11-19T05:25:09,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/5841153f4701410fb625d28ac91eb862 is 50, key is test_row_0/C:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:09,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742070_1246 (size=12001) 2024-11-19T05:25:09,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:09,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993969337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:09,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:09,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993969337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993969337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:09,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993969337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:09,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993969347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:09,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/5841153f4701410fb625d28ac91eb862 2024-11-19T05:25:09,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/bb9e7f968b1941f2ab1d28b5741cfb75 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75 2024-11-19T05:25:09,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75, entries=200, sequenceid=52, filesize=14.0 K 2024-11-19T05:25:09,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5c82b8de53124db8879031d2d44eb9f1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1 2024-11-19T05:25:09,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T05:25:09,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/5841153f4701410fb625d28ac91eb862 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862 2024-11-19T05:25:09,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862, entries=150, sequenceid=52, filesize=11.7 K 2024-11-19T05:25:09,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1324ms, sequenceid=52, compaction requested=true 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:09,486 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:09,486 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:09,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:09,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:09,487 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:09,487 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:09,487 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,487 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=39.7 K 2024-11-19T05:25:09,488 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:09,488 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:09,488 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,488 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.2 K 2024-11-19T05:25:09,488 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f16f5a95435649b8b2f4b8d177bac660, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993906815 2024-11-19T05:25:09,488 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting da448e01c91f4be5bd79448c7632f95c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993906815 2024-11-19T05:25:09,489 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c3109d0f654b2ea58d34b9d92232ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993907518 2024-11-19T05:25:09,489 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89fc1de128c74d598dfe3010dc8506b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993907518 2024-11-19T05:25:09,489 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c82b8de53124db8879031d2d44eb9f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:09,489 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb9e7f968b1941f2ab1d28b5741cfb75, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:09,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-19T05:25:09,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,493 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:25:09,499 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:09,500 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/ea7e23e5415342c396dd595707998fde is 50, key is test_row_0/B:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/9291823375c948b5bc56e7966878ddbe is 50, key is test_row_0/A:col10/1731993908187/Put/seqid=0 2024-11-19T05:25:09,505 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:09,505 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/c6e6ccb35fc74503ad19e30959baaefa is 50, key is test_row_0/A:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:09,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742071_1247 (size=12104) 2024-11-19T05:25:09,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742072_1248 (size=12104) 2024-11-19T05:25:09,542 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/c6e6ccb35fc74503ad19e30959baaefa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/c6e6ccb35fc74503ad19e30959baaefa 2024-11-19T05:25:09,548 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into c6e6ccb35fc74503ad19e30959baaefa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:09,549 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:09,549 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993909486; duration=0sec 2024-11-19T05:25:09,549 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:09,549 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:09,549 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:09,550 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:09,550 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:09,551 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:09,551 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.2 K 2024-11-19T05:25:09,552 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21605b7694164b08ba99b99c20b3d6d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993906815 2024-11-19T05:25:09,552 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62276e49c2874e70a21f253c1d01bcbc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993907518 2024-11-19T05:25:09,552 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5841153f4701410fb625d28ac91eb862, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:09,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:09,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/04aefedcc2334d709ca127d5b5e1f878 is 50, key is test_row_0/C:col10/1731993908159/Put/seqid=0 2024-11-19T05:25:09,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742073_1249 (size=12001) 2024-11-19T05:25:09,578 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/9291823375c948b5bc56e7966878ddbe 2024-11-19T05:25:09,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/194ffed3e8724684a053f0b4e9672138 is 50, key is test_row_0/B:col10/1731993908187/Put/seqid=0 2024-11-19T05:25:09,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742074_1250 (size=12104) 2024-11-19T05:25:09,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742075_1251 (size=12001) 2024-11-19T05:25:09,618 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/04aefedcc2334d709ca127d5b5e1f878 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/04aefedcc2334d709ca127d5b5e1f878 2024-11-19T05:25:09,625 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 04aefedcc2334d709ca127d5b5e1f878(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:09,625 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:09,625 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993909486; duration=0sec 2024-11-19T05:25:09,625 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:09,625 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:09,777 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:25:09,922 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/ea7e23e5415342c396dd595707998fde as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/ea7e23e5415342c396dd595707998fde 2024-11-19T05:25:09,930 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into ea7e23e5415342c396dd595707998fde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:09,930 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:09,930 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993909486; duration=0sec 2024-11-19T05:25:09,930 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:09,930 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:10,021 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/194ffed3e8724684a053f0b4e9672138 2024-11-19T05:25:10,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/463667c876e34c61a611f88bb2c6cc46 is 50, key is test_row_0/C:col10/1731993908187/Put/seqid=0 2024-11-19T05:25:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742076_1252 (size=12001) 2024-11-19T05:25:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:10,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:10,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993970357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993970358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,360 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993970359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993970360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993970367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,368 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,462 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/463667c876e34c61a611f88bb2c6cc46 2024-11-19T05:25:10,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/9291823375c948b5bc56e7966878ddbe as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe 2024-11-19T05:25:10,469 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:25:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993970467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/194ffed3e8724684a053f0b4e9672138 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138 2024-11-19T05:25:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993970467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993970467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,474 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:25:10,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/463667c876e34c61a611f88bb2c6cc46 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46 2024-11-19T05:25:10,478 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:25:10,479 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 7de0717bd410d1dc7ce54e46b0058ca3 in 986ms, sequenceid=77, compaction requested=false 2024-11-19T05:25:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-19T05:25:10,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-19T05:25:10,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-19T05:25:10,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6630 sec 2024-11-19T05:25:10,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.6700 sec 2024-11-19T05:25:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:10,678 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:10,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:25:10,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e373fe4709bc40fc8c71da67b5b9d7e9 is 50, key is test_row_0/A:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:10,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742077_1253 (size=9657) 2024-11-19T05:25:10,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993970702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993970706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993970707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993970817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993970817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993970817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:10,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-19T05:25:10,988 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:10,988 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-19T05:25:10,991 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:10,992 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:10,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:10,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T05:25:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993971028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993971028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993971028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T05:25:11,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e373fe4709bc40fc8c71da67b5b9d7e9 2024-11-19T05:25:11,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5778955e82744ad99b28e537230c16b1 is 50, key is test_row_0/B:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:11,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T05:25:11,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:11,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:11,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742078_1254 (size=9657) 2024-11-19T05:25:11,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5778955e82744ad99b28e537230c16b1 2024-11-19T05:25:11,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/12e71160f6a54530818d740d9710b9b7 is 50, key is test_row_0/C:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742079_1255 (size=9657) 2024-11-19T05:25:11,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/12e71160f6a54530818d740d9710b9b7 2024-11-19T05:25:11,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e373fe4709bc40fc8c71da67b5b9d7e9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9 2024-11-19T05:25:11,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9, entries=100, sequenceid=93, filesize=9.4 K 2024-11-19T05:25:11,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/5778955e82744ad99b28e537230c16b1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1 2024-11-19T05:25:11,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1, entries=100, sequenceid=93, filesize=9.4 K 2024-11-19T05:25:11,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/12e71160f6a54530818d740d9710b9b7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7 2024-11-19T05:25:11,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7, entries=100, sequenceid=93, filesize=9.4 K 2024-11-19T05:25:11,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 7de0717bd410d1dc7ce54e46b0058ca3 in 552ms, sequenceid=93, compaction requested=true 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:11,230 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:11,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:25:11,230 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:11,232 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:11,232 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:11,232 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,232 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/c6e6ccb35fc74503ad19e30959baaefa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=33.0 K 2024-11-19T05:25:11,232 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6e6ccb35fc74503ad19e30959baaefa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:11,234 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:11,234 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:11,234 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,234 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/ea7e23e5415342c396dd595707998fde, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=33.0 K 2024-11-19T05:25:11,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9291823375c948b5bc56e7966878ddbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993908187 2024-11-19T05:25:11,235 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ea7e23e5415342c396dd595707998fde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:11,237 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 194ffed3e8724684a053f0b4e9672138, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993908187 2024-11-19T05:25:11,237 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e373fe4709bc40fc8c71da67b5b9d7e9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993910358 2024-11-19T05:25:11,237 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5778955e82744ad99b28e537230c16b1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993910358 2024-11-19T05:25:11,262 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:11,263 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/55bfcb4b516546beb0f1a8066a63446c is 50, key is test_row_0/B:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:11,274 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:11,275 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/f1d2afb80365475e859174d2cc59c8fc is 50, key is test_row_0/A:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:11,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T05:25:11,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-19T05:25:11,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,308 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:25:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742080_1256 (size=12207) 2024-11-19T05:25:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:11,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:11,343 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/55bfcb4b516546beb0f1a8066a63446c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/55bfcb4b516546beb0f1a8066a63446c 2024-11-19T05:25:11,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e59a98950fa94af08d47581b59a29e6b is 50, key is test_row_0/A:col10/1731993910701/Put/seqid=0 2024-11-19T05:25:11,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742081_1257 (size=12207) 2024-11-19T05:25:11,359 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into 55bfcb4b516546beb0f1a8066a63446c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:11,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:11,359 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993911230; duration=0sec 2024-11-19T05:25:11,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:11,359 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:11,360 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:11,362 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:11,362 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:11,362 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,363 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/04aefedcc2334d709ca127d5b5e1f878, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=33.0 K 2024-11-19T05:25:11,363 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 04aefedcc2334d709ca127d5b5e1f878, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731993907522 2024-11-19T05:25:11,363 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/f1d2afb80365475e859174d2cc59c8fc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f1d2afb80365475e859174d2cc59c8fc 2024-11-19T05:25:11,363 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 463667c876e34c61a611f88bb2c6cc46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993908187 2024-11-19T05:25:11,364 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 12e71160f6a54530818d740d9710b9b7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993910358 2024-11-19T05:25:11,368 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into f1d2afb80365475e859174d2cc59c8fc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:11,368 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:11,368 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993911230; duration=0sec 2024-11-19T05:25:11,368 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:11,368 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:11,385 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:11,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/e39bd42fb5bf403f85f93668fef1738c is 50, key is test_row_0/C:col10/1731993910678/Put/seqid=0 2024-11-19T05:25:11,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993971377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993971378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993971377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742082_1258 (size=12001) 2024-11-19T05:25:11,411 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e59a98950fa94af08d47581b59a29e6b 2024-11-19T05:25:11,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/aec574d8207b40aa835803519b9ea779 is 50, key is test_row_0/B:col10/1731993910701/Put/seqid=0 2024-11-19T05:25:11,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742083_1259 (size=12207) 2024-11-19T05:25:11,446 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/e39bd42fb5bf403f85f93668fef1738c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e39bd42fb5bf403f85f93668fef1738c 2024-11-19T05:25:11,453 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into e39bd42fb5bf403f85f93668fef1738c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:11,453 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:11,453 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993911230; duration=0sec 2024-11-19T05:25:11,453 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:11,454 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:11,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742084_1260 (size=12001) 2024-11-19T05:25:11,459 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/aec574d8207b40aa835803519b9ea779 2024-11-19T05:25:11,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/24a3104a24024b1e9703bfbf8a792b11 is 50, key is test_row_0/C:col10/1731993910701/Put/seqid=0 2024-11-19T05:25:11,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742085_1261 (size=12001) 2024-11-19T05:25:11,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993971497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993971497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993971497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T05:25:11,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993971707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993971707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993971707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:11,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:11,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:11,887 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/24a3104a24024b1e9703bfbf8a792b11 2024-11-19T05:25:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/e59a98950fa94af08d47581b59a29e6b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b 2024-11-19T05:25:11,895 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T05:25:11,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/aec574d8207b40aa835803519b9ea779 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779 2024-11-19T05:25:11,905 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T05:25:11,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/24a3104a24024b1e9703bfbf8a792b11 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11 2024-11-19T05:25:11,911 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11, entries=150, sequenceid=116, filesize=11.7 K 2024-11-19T05:25:11,912 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 7de0717bd410d1dc7ce54e46b0058ca3 in 604ms, sequenceid=116, compaction requested=false 2024-11-19T05:25:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-19T05:25:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-19T05:25:11,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-19T05:25:11,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 923 msec 2024-11-19T05:25:11,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 931 msec 2024-11-19T05:25:12,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:12,020 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:12,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:25:12,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0c24a6d9e59a410dbb7a5ee101b5dbfc is 50, key is test_row_0/A:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742086_1262 (size=12101) 2024-11-19T05:25:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993972082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993972082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993972082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-19T05:25:12,128 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:12,129 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-19T05:25:12,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:12,132 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:12,132 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:12,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993972197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993972197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993972197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:12,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T05:25:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993972368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,370 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993972377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,381 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993972407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993972407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993972407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T05:25:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:12,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0c24a6d9e59a410dbb7a5ee101b5dbfc 2024-11-19T05:25:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:12,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8b8d84d7914347ef8806ff3ac6b2d70b is 50, key is test_row_0/B:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742087_1263 (size=12101) 2024-11-19T05:25:12,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8b8d84d7914347ef8806ff3ac6b2d70b 2024-11-19T05:25:12,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/f31f2d7b29b442c9996f3ab1dc42e9b7 is 50, key is test_row_0/C:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742088_1264 (size=12101) 2024-11-19T05:25:12,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/f31f2d7b29b442c9996f3ab1dc42e9b7 2024-11-19T05:25:12,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0c24a6d9e59a410dbb7a5ee101b5dbfc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc 2024-11-19T05:25:12,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T05:25:12,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8b8d84d7914347ef8806ff3ac6b2d70b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b 2024-11-19T05:25:12,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T05:25:12,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/f31f2d7b29b442c9996f3ab1dc42e9b7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7 2024-11-19T05:25:12,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7, entries=150, sequenceid=134, filesize=11.8 K 2024-11-19T05:25:12,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 7de0717bd410d1dc7ce54e46b0058ca3 in 524ms, sequenceid=134, compaction requested=true 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:12,544 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:12,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:25:12,544 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:12,546 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:12,546 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:12,546 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,546 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/55bfcb4b516546beb0f1a8066a63446c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.5 K 2024-11-19T05:25:12,546 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:12,546 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:12,547 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,547 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f1d2afb80365475e859174d2cc59c8fc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.5 K 2024-11-19T05:25:12,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 55bfcb4b516546beb0f1a8066a63446c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993908195 2024-11-19T05:25:12,547 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1d2afb80365475e859174d2cc59c8fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993908195 2024-11-19T05:25:12,548 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59a98950fa94af08d47581b59a29e6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993910699 2024-11-19T05:25:12,548 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting aec574d8207b40aa835803519b9ea779, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993910699 2024-11-19T05:25:12,548 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c24a6d9e59a410dbb7a5ee101b5dbfc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:12,548 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b8d84d7914347ef8806ff3ac6b2d70b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:12,566 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:12,567 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/fe31a0d5e9a342e3848135968541534c is 50, key is test_row_0/A:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,582 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:12,584 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/bf5bde88bbc0450497cd3725321e92ef is 50, key is test_row_0/B:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-19T05:25:12,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,592 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:25:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/17341a6d002746d9a0366b9e7bc08e64 is 50, key is test_row_0/A:col10/1731993912059/Put/seqid=0 2024-11-19T05:25:12,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742089_1265 (size=12409) 2024-11-19T05:25:12,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742090_1266 (size=12409) 2024-11-19T05:25:12,632 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/bf5bde88bbc0450497cd3725321e92ef as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/bf5bde88bbc0450497cd3725321e92ef 2024-11-19T05:25:12,640 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into bf5bde88bbc0450497cd3725321e92ef(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:12,640 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:12,640 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993912544; duration=0sec 2024-11-19T05:25:12,640 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:12,640 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:12,640 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:12,641 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:12,641 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:12,641 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:12,642 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e39bd42fb5bf403f85f93668fef1738c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.5 K 2024-11-19T05:25:12,642 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e39bd42fb5bf403f85f93668fef1738c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731993908195 2024-11-19T05:25:12,643 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 24a3104a24024b1e9703bfbf8a792b11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731993910699 2024-11-19T05:25:12,644 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f31f2d7b29b442c9996f3ab1dc42e9b7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742091_1267 (size=12151) 2024-11-19T05:25:12,657 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/17341a6d002746d9a0366b9e7bc08e64 2024-11-19T05:25:12,670 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:12,670 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/44b47fa46c9f4e0fbf6d82cf4fb78a06 is 50, key is test_row_0/C:col10/1731993912019/Put/seqid=0 2024-11-19T05:25:12,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/e4dfc6be431c4120ae3894d404838ecd is 50, key is test_row_0/B:col10/1731993912059/Put/seqid=0 2024-11-19T05:25:12,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742092_1268 (size=12409) 2024-11-19T05:25:12,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:12,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742093_1269 (size=12151) 2024-11-19T05:25:12,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993972742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993972742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993972744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993972857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993972857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993972857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,028 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/fe31a0d5e9a342e3848135968541534c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/fe31a0d5e9a342e3848135968541534c 2024-11-19T05:25:13,035 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into fe31a0d5e9a342e3848135968541534c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:13,035 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:13,035 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993912544; duration=0sec 2024-11-19T05:25:13,035 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:13,035 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993973067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993973067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993973068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,117 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/44b47fa46c9f4e0fbf6d82cf4fb78a06 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/44b47fa46c9f4e0fbf6d82cf4fb78a06 2024-11-19T05:25:13,120 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 44b47fa46c9f4e0fbf6d82cf4fb78a06(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:13,120 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:13,121 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993912544; duration=0sec 2024-11-19T05:25:13,121 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:13,121 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:13,138 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/e4dfc6be431c4120ae3894d404838ecd 2024-11-19T05:25:13,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/956bb99e044448eb90f99a6e8c910913 is 50, key is test_row_0/C:col10/1731993912059/Put/seqid=0 2024-11-19T05:25:13,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742094_1270 (size=12151) 2024-11-19T05:25:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:13,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993973377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993973377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993973378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,549 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/956bb99e044448eb90f99a6e8c910913 2024-11-19T05:25:13,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/17341a6d002746d9a0366b9e7bc08e64 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64 2024-11-19T05:25:13,557 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:25:13,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/e4dfc6be431c4120ae3894d404838ecd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd 2024-11-19T05:25:13,570 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:25:13,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/956bb99e044448eb90f99a6e8c910913 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913 2024-11-19T05:25:13,575 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:25:13,576 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 7de0717bd410d1dc7ce54e46b0058ca3 in 984ms, sequenceid=156, compaction requested=false 2024-11-19T05:25:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:13,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-19T05:25:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-19T05:25:13,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-19T05:25:13,580 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4460 sec 2024-11-19T05:25:13,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4520 sec 2024-11-19T05:25:13,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:13,889 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:13,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:25:13,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/30ce08ff9c72463c9015c30a1f1d4586 is 50, key is test_row_0/A:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:13,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742095_1271 (size=12151) 2024-11-19T05:25:13,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/30ce08ff9c72463c9015c30a1f1d4586 2024-11-19T05:25:13,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993973916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993973917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993973918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:13,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/a864661e6fce4f6cb452845a68638ce7 is 50, key is test_row_0/B:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:13,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742096_1272 (size=12151) 2024-11-19T05:25:14,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993974027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993974027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993974027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,031 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993974236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993974237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993974237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-19T05:25:14,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:14,278 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-19T05:25:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:14,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:14,280 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:14,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:14,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/a864661e6fce4f6cb452845a68638ce7 2024-11-19T05:25:14,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/60b77b016b794de08d2926027be722f7 is 50, key is test_row_0/C:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:14,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742097_1273 (size=12151) 2024-11-19T05:25:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:14,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T05:25:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:14,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993974547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993974548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993974548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:14,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:14,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T05:25:14,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:14,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:14,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T05:25:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:14,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/60b77b016b794de08d2926027be722f7 2024-11-19T05:25:14,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/30ce08ff9c72463c9015c30a1f1d4586 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586 2024-11-19T05:25:14,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T05:25:14,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/a864661e6fce4f6cb452845a68638ce7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7 2024-11-19T05:25:14,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T05:25:14,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/60b77b016b794de08d2926027be722f7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7 2024-11-19T05:25:14,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7, entries=150, sequenceid=176, filesize=11.9 K 2024-11-19T05:25:14,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 7de0717bd410d1dc7ce54e46b0058ca3 in 889ms, sequenceid=176, compaction requested=true 2024-11-19T05:25:14,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:14,779 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:14,779 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:14,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:14,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:14,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:14,780 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,780 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/bf5bde88bbc0450497cd3725321e92ef, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.9 K 2024-11-19T05:25:14,780 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:14,780 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:14,780 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,780 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/fe31a0d5e9a342e3848135968541534c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.9 K 2024-11-19T05:25:14,782 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe31a0d5e9a342e3848135968541534c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:14,782 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting bf5bde88bbc0450497cd3725321e92ef, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:14,782 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17341a6d002746d9a0366b9e7bc08e64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993912047 2024-11-19T05:25:14,783 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e4dfc6be431c4120ae3894d404838ecd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993912047 2024-11-19T05:25:14,783 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30ce08ff9c72463c9015c30a1f1d4586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:14,783 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a864661e6fce4f6cb452845a68638ce7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:14,810 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:14,811 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/901a602aedd748a7b4a50e6b7460055c is 50, key is test_row_0/A:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:14,821 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:14,821 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/0b1faf4173f44eacad28463e972bbc1b is 50, key is test_row_0/B:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:14,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742098_1274 (size=12561) 2024-11-19T05:25:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742099_1275 (size=12561) 2024-11-19T05:25:14,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-19T05:25:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:14,893 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T05:25:14,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b5c6d0e82df4285ab2c60714a1544fc is 50, key is test_row_0/A:col10/1731993913905/Put/seqid=0 2024-11-19T05:25:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:14,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742100_1276 (size=12151) 2024-11-19T05:25:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:15,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:15,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993975122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993975127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993975130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,229 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/901a602aedd748a7b4a50e6b7460055c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/901a602aedd748a7b4a50e6b7460055c 2024-11-19T05:25:15,231 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/0b1faf4173f44eacad28463e972bbc1b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/0b1faf4173f44eacad28463e972bbc1b 2024-11-19T05:25:15,234 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into 901a602aedd748a7b4a50e6b7460055c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:15,234 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:15,235 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993914779; duration=0sec 2024-11-19T05:25:15,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:15,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:15,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:15,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:15,236 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:15,236 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:15,236 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/44b47fa46c9f4e0fbf6d82cf4fb78a06, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=35.9 K 2024-11-19T05:25:15,236 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44b47fa46c9f4e0fbf6d82cf4fb78a06, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731993911343 2024-11-19T05:25:15,238 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 956bb99e044448eb90f99a6e8c910913, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993912047 2024-11-19T05:25:15,238 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into 0b1faf4173f44eacad28463e972bbc1b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:15,238 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:15,238 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60b77b016b794de08d2926027be722f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:15,238 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993914779; duration=0sec 2024-11-19T05:25:15,238 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:15,238 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993975237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993975237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,247 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:15,248 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/094d409951074ee4b145be4b9065ddb8 is 50, key is test_row_0/C:col10/1731993913888/Put/seqid=0 2024-11-19T05:25:15,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993975248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742101_1277 (size=12561) 2024-11-19T05:25:15,310 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b5c6d0e82df4285ab2c60714a1544fc 2024-11-19T05:25:15,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/df30360f22a6443c8ef57390ce45a59b is 50, key is test_row_0/B:col10/1731993913905/Put/seqid=0 2024-11-19T05:25:15,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742102_1278 (size=12151) 2024-11-19T05:25:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:15,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993975447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993975447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993975457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,660 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/094d409951074ee4b145be4b9065ddb8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/094d409951074ee4b145be4b9065ddb8 2024-11-19T05:25:15,666 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 094d409951074ee4b145be4b9065ddb8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:15,667 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:15,667 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993914779; duration=0sec 2024-11-19T05:25:15,667 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:15,667 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:15,721 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/df30360f22a6443c8ef57390ce45a59b 2024-11-19T05:25:15,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/b396a59bc2a345d389aa5064be9a4073 is 50, key is test_row_0/C:col10/1731993913905/Put/seqid=0 2024-11-19T05:25:15,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742103_1279 (size=12151) 2024-11-19T05:25:15,738 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/b396a59bc2a345d389aa5064be9a4073 2024-11-19T05:25:15,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b5c6d0e82df4285ab2c60714a1544fc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc 2024-11-19T05:25:15,745 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T05:25:15,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/df30360f22a6443c8ef57390ce45a59b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b 2024-11-19T05:25:15,751 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T05:25:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/b396a59bc2a345d389aa5064be9a4073 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073 2024-11-19T05:25:15,757 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073, entries=150, sequenceid=195, filesize=11.9 K 2024-11-19T05:25:15,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 7de0717bd410d1dc7ce54e46b0058ca3 in 864ms, sequenceid=195, compaction requested=false 2024-11-19T05:25:15,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:15,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:15,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-19T05:25:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-19T05:25:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:15,760 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:15,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T05:25:15,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-19T05:25:15,761 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-19T05:25:15,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.4840 sec 2024-11-19T05:25:15,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/5b8a99b2f14941c88d1c6dd2926e3c07 is 50, key is test_row_0/A:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:15,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742104_1280 (size=12151) 2024-11-19T05:25:15,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993975830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993975831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993975832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993975937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993975947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:15,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993975947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:15,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993976147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993976157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993976157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/5b8a99b2f14941c88d1c6dd2926e3c07 2024-11-19T05:25:16,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/445c63107a1e4b81bc4037d4a7d2a7be is 50, key is test_row_0/B:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:16,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742105_1281 (size=12151) 2024-11-19T05:25:16,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993976397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,398 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993976397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,400 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-19T05:25:16,427 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:16,428 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:16,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-19T05:25:16,429 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:16,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:16,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:16,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:16,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993976457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993976467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993976467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:16,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T05:25:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:16,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/445c63107a1e4b81bc4037d4a7d2a7be 2024-11-19T05:25:16,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/8c5baea1a6d741c7af5a49ad128fe4df is 50, key is test_row_0/C:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:16,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742106_1282 (size=12151) 2024-11-19T05:25:16,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T05:25:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:16,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T05:25:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:16,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993976967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993976977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:16,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993976977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:16,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:17,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T05:25:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:17,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:17,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/8c5baea1a6d741c7af5a49ad128fe4df 2024-11-19T05:25:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:17,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/5b8a99b2f14941c88d1c6dd2926e3c07 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07 2024-11-19T05:25:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T05:25:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/445c63107a1e4b81bc4037d4a7d2a7be as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be 2024-11-19T05:25:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:17,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T05:25:17,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/8c5baea1a6d741c7af5a49ad128fe4df as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df 2024-11-19T05:25:17,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df, entries=150, sequenceid=216, filesize=11.9 K 2024-11-19T05:25:17,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1305ms, sequenceid=216, compaction requested=true 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:17,065 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:17,065 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:17,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:17,066 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,066 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,066 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/901a602aedd748a7b4a50e6b7460055c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.0 K 2024-11-19T05:25:17,066 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/0b1faf4173f44eacad28463e972bbc1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.0 K 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1faf4173f44eacad28463e972bbc1b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:17,066 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 901a602aedd748a7b4a50e6b7460055c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:17,067 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5c6d0e82df4285ab2c60714a1544fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731993913905 2024-11-19T05:25:17,067 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting df30360f22a6443c8ef57390ce45a59b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731993913905 2024-11-19T05:25:17,067 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b8a99b2f14941c88d1c6dd2926e3c07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:17,067 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 445c63107a1e4b81bc4037d4a7d2a7be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:17,078 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:17,078 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/be9033e8415d4aae96a5720479efde7f is 50, key is test_row_0/B:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:17,094 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:17,094 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/35f00535b80f411791189369b5be9181 is 50, key is test_row_0/A:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:17,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742107_1283 (size=12663) 2024-11-19T05:25:17,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742108_1284 (size=12663) 2024-11-19T05:25:17,119 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/35f00535b80f411791189369b5be9181 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/35f00535b80f411791189369b5be9181 2024-11-19T05:25:17,124 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into 35f00535b80f411791189369b5be9181(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:17,124 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:17,124 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993917065; duration=0sec 2024-11-19T05:25:17,124 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:17,124 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:17,124 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:17,126 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:17,126 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:17,126 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,126 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/094d409951074ee4b145be4b9065ddb8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.0 K 2024-11-19T05:25:17,126 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 094d409951074ee4b145be4b9065ddb8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731993912733 2024-11-19T05:25:17,126 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b396a59bc2a345d389aa5064be9a4073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731993913905 2024-11-19T05:25:17,127 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c5baea1a6d741c7af5a49ad128fe4df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:17,139 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:17,140 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/824bc8c77e3641bba7cabc949427dcf0 is 50, key is test_row_0/C:col10/1731993915119/Put/seqid=0 2024-11-19T05:25:17,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742109_1285 (size=12663) 2024-11-19T05:25:17,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-19T05:25:17,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:17,197 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-19T05:25:17,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/d8c1bbe2892a4b628f9965d1586a709e is 50, key is test_row_0/A:col10/1731993915830/Put/seqid=0 2024-11-19T05:25:17,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742110_1286 (size=12151) 2024-11-19T05:25:17,207 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/d8c1bbe2892a4b628f9965d1586a709e 2024-11-19T05:25:17,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/766e2c8267b648a58cf014c19aa6cc79 is 50, key is test_row_0/B:col10/1731993915830/Put/seqid=0 2024-11-19T05:25:17,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742111_1287 (size=12151) 2024-11-19T05:25:17,510 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/be9033e8415d4aae96a5720479efde7f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/be9033e8415d4aae96a5720479efde7f 2024-11-19T05:25:17,513 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into be9033e8415d4aae96a5720479efde7f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:17,513 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:17,513 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993917065; duration=0sec 2024-11-19T05:25:17,514 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:17,514 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:17,564 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/824bc8c77e3641bba7cabc949427dcf0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/824bc8c77e3641bba7cabc949427dcf0 2024-11-19T05:25:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:17,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 824bc8c77e3641bba7cabc949427dcf0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:17,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:17,568 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993917065; duration=0sec 2024-11-19T05:25:17,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:17,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:17,621 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/766e2c8267b648a58cf014c19aa6cc79 2024-11-19T05:25:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/1a19db3663af42c2a6aefd4028561414 is 50, key is test_row_0/C:col10/1731993915830/Put/seqid=0 2024-11-19T05:25:17,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742112_1288 (size=12151) 2024-11-19T05:25:17,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:17,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:18,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993978013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993978016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993978016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,038 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/1a19db3663af42c2a6aefd4028561414 2024-11-19T05:25:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/d8c1bbe2892a4b628f9965d1586a709e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e 2024-11-19T05:25:18,045 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T05:25:18,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/766e2c8267b648a58cf014c19aa6cc79 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79 2024-11-19T05:25:18,049 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T05:25:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/1a19db3663af42c2a6aefd4028561414 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414 2024-11-19T05:25:18,053 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T05:25:18,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 7de0717bd410d1dc7ce54e46b0058ca3 in 857ms, sequenceid=235, compaction requested=false 2024-11-19T05:25:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:18,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-19T05:25:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-19T05:25:18,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-19T05:25:18,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-19T05:25:18,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6300 sec 2024-11-19T05:25:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:18,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:25:18,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b7727f677d4459ba0ca784316af779b is 50, key is test_row_0/A:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:18,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742113_1289 (size=17031) 2024-11-19T05:25:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993978145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993978146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993978148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993978257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993978257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993978257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993978467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993978467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993978467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b7727f677d4459ba0ca784316af779b 2024-11-19T05:25:18,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/191e9b61c0b345769e6975eb1a425ef5 is 50, key is test_row_0/B:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:18,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742114_1290 (size=12201) 2024-11-19T05:25:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-19T05:25:18,577 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:18,578 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-19T05:25:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:18,579 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:18,580 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:18,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:18,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:18,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:18,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993978777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993978777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993978777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:18,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:18,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:18,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/191e9b61c0b345769e6975eb1a425ef5 2024-11-19T05:25:18,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/7623ca93050a49fca871f97b1d577d6f is 50, key is test_row_0/C:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742115_1291 (size=12201) 2024-11-19T05:25:19,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,038 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993979287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993979287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993979287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:19,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:19,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=258 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/7623ca93050a49fca871f97b1d577d6f 2024-11-19T05:25:19,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/2b7727f677d4459ba0ca784316af779b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b 2024-11-19T05:25:19,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b, entries=250, sequenceid=258, filesize=16.6 K 2024-11-19T05:25:19,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/191e9b61c0b345769e6975eb1a425ef5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5 2024-11-19T05:25:19,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T05:25:19,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/7623ca93050a49fca871f97b1d577d6f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f 2024-11-19T05:25:19,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f, entries=150, sequenceid=258, filesize=11.9 K 2024-11-19T05:25:19,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1247ms, sequenceid=258, compaction requested=true 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:19,376 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:19,376 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:19,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41845 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:19,377 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,377 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,377 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/be9033e8415d4aae96a5720479efde7f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.1 K 2024-11-19T05:25:19,377 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/35f00535b80f411791189369b5be9181, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=40.9 K 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting be9033e8415d4aae96a5720479efde7f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:19,377 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35f00535b80f411791189369b5be9181, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:19,378 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8c1bbe2892a4b628f9965d1586a709e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993915818 2024-11-19T05:25:19,378 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 766e2c8267b648a58cf014c19aa6cc79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993915818 2024-11-19T05:25:19,379 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 191e9b61c0b345769e6975eb1a425ef5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:19,379 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b7727f677d4459ba0ca784316af779b, keycount=250, bloomtype=ROW, size=16.6 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:19,385 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:19,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/4872674a36784d33b7499f1303fe7e37 is 50, key is test_row_0/B:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:19,385 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:19,386 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/03b2317ca7f047ce93e2ed01c826e070 is 50, key is test_row_0/A:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:19,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742117_1293 (size=12815) 2024-11-19T05:25:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742116_1292 (size=12815) 2024-11-19T05:25:19,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-19T05:25:19,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,496 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:25:19,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/725dd30b28cb4707b7aab41e703b6ddf is 50, key is test_row_0/A:col10/1731993918144/Put/seqid=0 2024-11-19T05:25:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742118_1294 (size=12301) 2024-11-19T05:25:19,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:19,795 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/03b2317ca7f047ce93e2ed01c826e070 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/03b2317ca7f047ce93e2ed01c826e070 2024-11-19T05:25:19,796 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/4872674a36784d33b7499f1303fe7e37 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/4872674a36784d33b7499f1303fe7e37 2024-11-19T05:25:19,800 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into 03b2317ca7f047ce93e2ed01c826e070(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:19,800 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into 4872674a36784d33b7499f1303fe7e37(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:19,800 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993919376; duration=0sec 2024-11-19T05:25:19,800 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993919376; duration=0sec 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:19,800 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:19,801 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37015 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:19,801 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:19,801 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:19,801 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/824bc8c77e3641bba7cabc949427dcf0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.1 K 2024-11-19T05:25:19,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 824bc8c77e3641bba7cabc949427dcf0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731993915119 2024-11-19T05:25:19,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a19db3663af42c2a6aefd4028561414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993915818 2024-11-19T05:25:19,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7623ca93050a49fca871f97b1d577d6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:19,809 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:19,810 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/024df7cc105047f5a52217068a5363ac is 50, key is test_row_0/C:col10/1731993918015/Put/seqid=0 2024-11-19T05:25:19,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742119_1295 (size=12815) 2024-11-19T05:25:19,904 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/725dd30b28cb4707b7aab41e703b6ddf 2024-11-19T05:25:19,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2da59bc890c5481c9066bf24e21e1fd4 is 50, key is test_row_0/B:col10/1731993918144/Put/seqid=0 2024-11-19T05:25:19,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742120_1296 (size=12301) 2024-11-19T05:25:20,235 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/024df7cc105047f5a52217068a5363ac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/024df7cc105047f5a52217068a5363ac 2024-11-19T05:25:20,239 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 024df7cc105047f5a52217068a5363ac(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:20,239 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:20,239 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993919376; duration=0sec 2024-11-19T05:25:20,239 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:20,239 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:20,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:20,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:20,318 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2da59bc890c5481c9066bf24e21e1fd4 2024-11-19T05:25:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6a05eb6f7c40430d926975422dc61c69 is 50, key is test_row_0/C:col10/1731993918144/Put/seqid=0 2024-11-19T05:25:20,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742121_1297 (size=12301) 2024-11-19T05:25:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993980324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993980325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993980327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993980437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993980437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993980437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993980648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993980648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993980648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,727 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6a05eb6f7c40430d926975422dc61c69 2024-11-19T05:25:20,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/725dd30b28cb4707b7aab41e703b6ddf as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf 2024-11-19T05:25:20,734 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf, entries=150, sequenceid=273, filesize=12.0 K 2024-11-19T05:25:20,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2da59bc890c5481c9066bf24e21e1fd4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4 2024-11-19T05:25:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:20,738 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4, entries=150, sequenceid=273, filesize=12.0 K 2024-11-19T05:25:20,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6a05eb6f7c40430d926975422dc61c69 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69 2024-11-19T05:25:20,741 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69, entries=150, sequenceid=273, filesize=12.0 K 2024-11-19T05:25:20,742 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1246ms, sequenceid=273, compaction requested=false 2024-11-19T05:25:20,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:20,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:20,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-19T05:25:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-19T05:25:20,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-19T05:25:20,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-19T05:25:20,746 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.1670 sec 2024-11-19T05:25:20,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:20,960 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:20,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:25:20,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/766c85f0faa945e7a4ffc0eb5ac658f5 is 50, key is test_row_0/A:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:20,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742122_1298 (size=12301) 2024-11-19T05:25:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993980976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993980976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:20,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:20,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993980981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:20,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993981087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993981087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993981097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993981297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993981297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993981307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/766c85f0faa945e7a4ffc0eb5ac658f5 2024-11-19T05:25:21,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2f829522b05940d382027f6ba83c6b71 is 50, key is test_row_0/B:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:21,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742123_1299 (size=12301) 2024-11-19T05:25:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993981607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993981607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993981617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:21,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:21,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2f829522b05940d382027f6ba83c6b71 2024-11-19T05:25:21,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/55916631dac8415a8bc3fd4809d90f83 is 50, key is test_row_0/C:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:21,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742124_1300 (size=12301) 2024-11-19T05:25:22,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993982117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:22,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993982117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:22,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993982127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:22,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:22,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/55916631dac8415a8bc3fd4809d90f83 2024-11-19T05:25:22,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/766c85f0faa945e7a4ffc0eb5ac658f5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5 2024-11-19T05:25:22,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T05:25:22,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/2f829522b05940d382027f6ba83c6b71 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71 2024-11-19T05:25:22,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T05:25:22,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/55916631dac8415a8bc3fd4809d90f83 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83 2024-11-19T05:25:22,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83, entries=150, sequenceid=298, filesize=12.0 K 2024-11-19T05:25:22,215 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1254ms, sequenceid=298, compaction requested=true 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:22,215 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:22,215 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:22,215 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:22,216 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:22,216 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:22,216 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:22,216 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:22,216 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:22,216 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:22,216 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/4872674a36784d33b7499f1303fe7e37, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.5 K 2024-11-19T05:25:22,216 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/03b2317ca7f047ce93e2ed01c826e070, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.5 K 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 4872674a36784d33b7499f1303fe7e37, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03b2317ca7f047ce93e2ed01c826e070, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725dd30b28cb4707b7aab41e703b6ddf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731993918143 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da59bc890c5481c9066bf24e21e1fd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731993918143 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 766c85f0faa945e7a4ffc0eb5ac658f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:22,217 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f829522b05940d382027f6ba83c6b71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:22,226 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:22,226 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:22,227 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fbd67c7c2ade4e1fa93e81a11176c8db is 50, key is test_row_0/B:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:22,227 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/cde52e7b451b4ded98cf40125193db53 is 50, key is test_row_0/A:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:22,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742125_1301 (size=13017) 2024-11-19T05:25:22,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742126_1302 (size=13017) 2024-11-19T05:25:22,238 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fbd67c7c2ade4e1fa93e81a11176c8db as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fbd67c7c2ade4e1fa93e81a11176c8db 2024-11-19T05:25:22,243 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/cde52e7b451b4ded98cf40125193db53 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/cde52e7b451b4ded98cf40125193db53 2024-11-19T05:25:22,248 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into fbd67c7c2ade4e1fa93e81a11176c8db(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:22,248 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:22,248 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993922215; duration=0sec 2024-11-19T05:25:22,248 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:22,248 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:22,248 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:22,250 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:22,250 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:22,250 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:22,250 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/024df7cc105047f5a52217068a5363ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.5 K 2024-11-19T05:25:22,251 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into cde52e7b451b4ded98cf40125193db53(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:22,251 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:22,251 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993922215; duration=0sec 2024-11-19T05:25:22,252 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:22,252 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:22,252 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 024df7cc105047f5a52217068a5363ac, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=258, earliestPutTs=1731993918008 2024-11-19T05:25:22,252 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a05eb6f7c40430d926975422dc61c69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731993918143 2024-11-19T05:25:22,252 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 55916631dac8415a8bc3fd4809d90f83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:22,260 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:22,261 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/f7062c50f0124a5c90c11eb1138d4b40 is 50, key is test_row_0/C:col10/1731993920324/Put/seqid=0 2024-11-19T05:25:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742127_1303 (size=13017) 2024-11-19T05:25:22,670 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/f7062c50f0124a5c90c11eb1138d4b40 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f7062c50f0124a5c90c11eb1138d4b40 2024-11-19T05:25:22,674 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into f7062c50f0124a5c90c11eb1138d4b40(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:22,674 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:22,674 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993922215; duration=0sec 2024-11-19T05:25:22,674 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:22,674 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-19T05:25:22,758 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:22,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:22,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-19T05:25:22,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:22,760 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:22,761 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:22,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:22,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-19T05:25:22,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:22,913 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:25:22,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/007516f228a94dc09cf998e86008fbc9 is 50, key is test_row_0/A:col10/1731993920973/Put/seqid=0 2024-11-19T05:25:22,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742128_1304 (size=12301) 2024-11-19T05:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993983158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993983158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993983161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993983267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993983267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993983267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,326 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/007516f228a94dc09cf998e86008fbc9 2024-11-19T05:25:23,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/889ec3a07ab742e29d17a133dbae9aba is 50, key is test_row_0/B:col10/1731993920973/Put/seqid=0 2024-11-19T05:25:23,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742129_1305 (size=12301) 2024-11-19T05:25:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:23,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993983477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993983477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993983477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,736 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/889ec3a07ab742e29d17a133dbae9aba 2024-11-19T05:25:23,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/29c04d8aa37a4756952c5121a3c50edf is 50, key is test_row_0/C:col10/1731993920973/Put/seqid=0 2024-11-19T05:25:23,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742130_1306 (size=12301) 2024-11-19T05:25:23,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993983787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993983787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993983787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:23,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:24,147 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/29c04d8aa37a4756952c5121a3c50edf 2024-11-19T05:25:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/007516f228a94dc09cf998e86008fbc9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9 2024-11-19T05:25:24,154 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T05:25:24,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/889ec3a07ab742e29d17a133dbae9aba as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba 2024-11-19T05:25:24,163 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T05:25:24,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/29c04d8aa37a4756952c5121a3c50edf as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf 2024-11-19T05:25:24,167 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf, entries=150, sequenceid=315, filesize=12.0 K 2024-11-19T05:25:24,168 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1255ms, sequenceid=315, compaction requested=false 2024-11-19T05:25:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-19T05:25:24,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-19T05:25:24,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-19T05:25:24,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-19T05:25:24,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4120 sec 2024-11-19T05:25:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:24,300 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:24,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:25:24,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/a8d5eaffd8e14156a29303a86f31fcdc is 50, key is test_row_0/A:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742131_1307 (size=14741) 2024-11-19T05:25:24,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993984315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993984316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993984317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993984427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993984427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993984427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993984637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993984637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993984637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/a8d5eaffd8e14156a29303a86f31fcdc 2024-11-19T05:25:24,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8a9987ec1b814d91b209c4ad3a0504bc is 50, key is test_row_0/B:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:24,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742132_1308 (size=12301) 2024-11-19T05:25:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-19T05:25:24,907 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:24,907 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-19T05:25:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:24,909 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:24,909 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:24,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993984947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993984947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993984947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:25,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T05:25:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8a9987ec1b814d91b209c4ad3a0504bc 2024-11-19T05:25:25,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/e702bf2c0c9046c49baa88d1f4d4c0cd is 50, key is test_row_0/C:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:25,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742133_1309 (size=12301) 2024-11-19T05:25:25,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T05:25:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:25,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:25,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T05:25:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993985457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993985457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:25,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993985457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:25,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:25,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T05:25:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:25,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/e702bf2c0c9046c49baa88d1f4d4c0cd 2024-11-19T05:25:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/a8d5eaffd8e14156a29303a86f31fcdc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc 2024-11-19T05:25:25,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc, entries=200, sequenceid=340, filesize=14.4 K 2024-11-19T05:25:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:25,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8a9987ec1b814d91b209c4ad3a0504bc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc 2024-11-19T05:25:25,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T05:25:25,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/e702bf2c0c9046c49baa88d1f4d4c0cd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd 2024-11-19T05:25:25,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd, entries=150, sequenceid=340, filesize=12.0 K 2024-11-19T05:25:25,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1245ms, sequenceid=340, compaction requested=true 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:25,545 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:25,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:25,545 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:25,546 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:25,546 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:25,546 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,547 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/cde52e7b451b4ded98cf40125193db53, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=39.1 K 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:25,547 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,547 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fbd67c7c2ade4e1fa93e81a11176c8db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.7 K 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting cde52e7b451b4ded98cf40125193db53, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fbd67c7c2ade4e1fa93e81a11176c8db, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 007516f228a94dc09cf998e86008fbc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731993920973 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8d5eaffd8e14156a29303a86f31fcdc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:25,547 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 889ec3a07ab742e29d17a133dbae9aba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731993920973 2024-11-19T05:25:25,548 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a9987ec1b814d91b209c4ad3a0504bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:25,553 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:25,553 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/aa8e8057c8fb4e3f814c7cb2e402c887 is 50, key is test_row_0/A:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742134_1310 (size=13119) 2024-11-19T05:25:25,558 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:25,558 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fa724a82d1c34e9daa2b56972f920f9a is 50, key is test_row_0/B:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:25,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742135_1311 (size=13119) 2024-11-19T05:25:25,568 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/aa8e8057c8fb4e3f814c7cb2e402c887 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/aa8e8057c8fb4e3f814c7cb2e402c887 2024-11-19T05:25:25,571 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into aa8e8057c8fb4e3f814c7cb2e402c887(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:25,571 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:25,571 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993925545; duration=0sec 2024-11-19T05:25:25,572 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:25,572 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:25,572 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:25,572 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:25,573 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:25,573 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,573 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f7062c50f0124a5c90c11eb1138d4b40, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.7 K 2024-11-19T05:25:25,573 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7062c50f0124a5c90c11eb1138d4b40, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731993920323 2024-11-19T05:25:25,573 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29c04d8aa37a4756952c5121a3c50edf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731993920973 2024-11-19T05:25:25,573 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e702bf2c0c9046c49baa88d1f4d4c0cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:25,586 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:25,587 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/123c73dfc1b04905be8b66bba8040e6a is 50, key is test_row_0/C:col10/1731993924300/Put/seqid=0 2024-11-19T05:25:25,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742136_1312 (size=13119) 2024-11-19T05:25:25,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-19T05:25:25,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:25,672 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:25:25,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/ee67dd917d2143b0aee31e0b698539da is 50, key is test_row_0/A:col10/1731993924308/Put/seqid=0 2024-11-19T05:25:25,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742137_1313 (size=12301) 2024-11-19T05:25:25,968 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fa724a82d1c34e9daa2b56972f920f9a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fa724a82d1c34e9daa2b56972f920f9a 2024-11-19T05:25:25,973 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into fa724a82d1c34e9daa2b56972f920f9a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:25,973 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:25,973 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993925545; duration=0sec 2024-11-19T05:25:25,973 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:25,973 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:25,998 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/123c73dfc1b04905be8b66bba8040e6a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/123c73dfc1b04905be8b66bba8040e6a 2024-11-19T05:25:26,002 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 123c73dfc1b04905be8b66bba8040e6a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:26,002 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:26,002 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993925545; duration=0sec 2024-11-19T05:25:26,003 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:26,003 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:26,080 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/ee67dd917d2143b0aee31e0b698539da 2024-11-19T05:25:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8aa1eff91de5467b98fe3345da1770b4 is 50, key is test_row_0/B:col10/1731993924308/Put/seqid=0 2024-11-19T05:25:26,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742138_1314 (size=12301) 2024-11-19T05:25:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:26,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. as already flushing 2024-11-19T05:25:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993986491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993986492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993986492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993986497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53402 deadline: 1731993986507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,509 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 7de0717bd410d1dc7ce54e46b0058ca3 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18308 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,522 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8aa1eff91de5467b98fe3345da1770b4 2024-11-19T05:25:26,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6ba64f97379241d7a220e43c8434915d is 50, key is test_row_0/C:col10/1731993924308/Put/seqid=0 2024-11-19T05:25:26,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742139_1315 (size=12301) 2024-11-19T05:25:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993986607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993986607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993986607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993986607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53400 deadline: 1731993986817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53442 deadline: 1731993986817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53470 deadline: 1731993986817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53460 deadline: 1731993986818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:25:26,866 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-19T05:25:26,866 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:26,866 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,866 INFO [Thread-1121 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1121. 2024-11-19T05:25:26,866 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,866 DEBUG [Thread-1121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:26,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:26,866 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,866 DEBUG [Thread-1121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,866 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:26,867 INFO [Thread-1123 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1123. 2024-11-19T05:25:26,867 DEBUG [Thread-1123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:26,867 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,867 DEBUG [Thread-1123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:26,869 INFO [Thread-1119 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1119. 2024-11-19T05:25:26,869 DEBUG [Thread-1119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:26,869 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,869 DEBUG [Thread-1119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:26,869 INFO [Thread-1125 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1125. 2024-11-19T05:25:26,869 DEBUG [Thread-1125 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:26,869 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,869 DEBUG [Thread-1125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:26,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:26,933 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6ba64f97379241d7a220e43c8434915d 2024-11-19T05:25:26,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/ee67dd917d2143b0aee31e0b698539da as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da 2024-11-19T05:25:26,942 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T05:25:26,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/8aa1eff91de5467b98fe3345da1770b4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4 2024-11-19T05:25:26,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T05:25:26,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/6ba64f97379241d7a220e43c8434915d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d 2024-11-19T05:25:26,952 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d, entries=150, sequenceid=353, filesize=12.0 K 2024-11-19T05:25:26,952 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1280ms, sequenceid=353, compaction requested=false 2024-11-19T05:25:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:26,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-19T05:25:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-19T05:25:26,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-19T05:25:26,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0450 sec 2024-11-19T05:25:26,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.0500 sec 2024-11-19T05:25:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-19T05:25:27,059 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:27,115 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:25:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:27,133 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:27,133 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-19T05:25:27,134 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-19T05:25:27,134 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-19T05:25:27,134 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-19T05:25:27,134 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:27,134 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:27,134 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,134 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:27,134 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,134 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,135 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,135 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,135 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:27,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:27,135 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:27,137 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-19T05:25:27,137 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:27,137 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,138 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:27,138 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:27,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/22f1376e93054ff7bae26dad552e2b22 is 50, key is test_row_0/A:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:27,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742140_1316 (size=12301) 2024-11-19T05:25:27,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/22f1376e93054ff7bae26dad552e2b22 2024-11-19T05:25:27,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7d1af9d9b21149a7ba984db7c5c18252 is 50, key is test_row_0/B:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:27,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742141_1317 (size=12301) 2024-11-19T05:25:27,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7d1af9d9b21149a7ba984db7c5c18252 2024-11-19T05:25:27,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/442d5f27cc434b97b589717de0779323 is 50, key is test_row_0/C:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:27,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742142_1318 (size=12301) 2024-11-19T05:25:28,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/442d5f27cc434b97b589717de0779323 2024-11-19T05:25:28,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/22f1376e93054ff7bae26dad552e2b22 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22 2024-11-19T05:25:28,404 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22, entries=150, sequenceid=382, filesize=12.0 K 2024-11-19T05:25:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7d1af9d9b21149a7ba984db7c5c18252 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252 2024-11-19T05:25:28,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252, entries=150, sequenceid=382, filesize=12.0 K 2024-11-19T05:25:28,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/442d5f27cc434b97b589717de0779323 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323 2024-11-19T05:25:28,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323, entries=150, sequenceid=382, filesize=12.0 K 2024-11-19T05:25:28,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=6.71 KB/6870 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1284ms, sequenceid=382, compaction requested=true 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:28,417 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7de0717bd410d1dc7ce54e46b0058ca3:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:28,417 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:28,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:28,418 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:28,418 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/B is initiating minor compaction (all files) 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/A is initiating minor compaction (all files) 2024-11-19T05:25:28,419 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/A in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:28,419 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/B in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:28,419 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/aa8e8057c8fb4e3f814c7cb2e402c887, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.8 K 2024-11-19T05:25:28,419 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fa724a82d1c34e9daa2b56972f920f9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.8 K 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fa724a82d1c34e9daa2b56972f920f9a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8e8057c8fb4e3f814c7cb2e402c887, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa1eff91de5467b98fe3345da1770b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731993924308 2024-11-19T05:25:28,419 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee67dd917d2143b0aee31e0b698539da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731993924308 2024-11-19T05:25:28,420 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d1af9d9b21149a7ba984db7c5c18252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731993926496 2024-11-19T05:25:28,420 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f1376e93054ff7bae26dad552e2b22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731993926496 2024-11-19T05:25:28,427 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#B#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:28,428 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fdb4403921784188ab81d384daad6ece is 50, key is test_row_0/B:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:28,428 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:28,429 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/4df92a04e90446aa89f08d7343a28558 is 50, key is test_row_0/A:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:28,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742143_1319 (size=13221) 2024-11-19T05:25:28,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742144_1320 (size=13221) 2024-11-19T05:25:28,841 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/fdb4403921784188ab81d384daad6ece as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fdb4403921784188ab81d384daad6ece 2024-11-19T05:25:28,842 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/4df92a04e90446aa89f08d7343a28558 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/4df92a04e90446aa89f08d7343a28558 2024-11-19T05:25:28,847 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/B of 7de0717bd410d1dc7ce54e46b0058ca3 into fdb4403921784188ab81d384daad6ece(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:28,847 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:28,847 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/B, priority=13, startTime=1731993928417; duration=0sec 2024-11-19T05:25:28,847 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:28,847 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/A of 7de0717bd410d1dc7ce54e46b0058ca3 into 4df92a04e90446aa89f08d7343a28558(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:28,847 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:B 2024-11-19T05:25:28,847 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:28,847 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/A, priority=13, startTime=1731993928417; duration=0sec 2024-11-19T05:25:28,847 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:28,848 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:28,848 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:A 2024-11-19T05:25:28,848 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:28,848 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 7de0717bd410d1dc7ce54e46b0058ca3/C is initiating minor compaction (all files) 2024-11-19T05:25:28,849 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7de0717bd410d1dc7ce54e46b0058ca3/C in TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:28,849 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/123c73dfc1b04905be8b66bba8040e6a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp, totalSize=36.8 K 2024-11-19T05:25:28,849 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 123c73dfc1b04905be8b66bba8040e6a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731993923157 2024-11-19T05:25:28,849 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ba64f97379241d7a220e43c8434915d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731993924308 2024-11-19T05:25:28,849 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 442d5f27cc434b97b589717de0779323, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731993926496 2024-11-19T05:25:28,855 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7de0717bd410d1dc7ce54e46b0058ca3#C#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:28,856 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/72862c3132f64757a4cdf1db20b01750 is 50, key is test_row_0/C:col10/1731993927129/Put/seqid=0 2024-11-19T05:25:28,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742145_1321 (size=13221) 2024-11-19T05:25:29,272 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/72862c3132f64757a4cdf1db20b01750 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/72862c3132f64757a4cdf1db20b01750 2024-11-19T05:25:29,280 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7de0717bd410d1dc7ce54e46b0058ca3/C of 7de0717bd410d1dc7ce54e46b0058ca3 into 72862c3132f64757a4cdf1db20b01750(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:29,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7de0717bd410d1dc7ce54e46b0058ca3: 2024-11-19T05:25:29,280 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., storeName=7de0717bd410d1dc7ce54e46b0058ca3/C, priority=13, startTime=1731993928417; duration=0sec 2024-11-19T05:25:29,280 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:29,281 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7de0717bd410d1dc7ce54e46b0058ca3:C 2024-11-19T05:25:36,578 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-19T05:25:36,579 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:25:36,579 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:36,579 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:36,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3877 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3794 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3885 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3890 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3769 2024-11-19T05:25:36,579 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:25:36,579 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:25:36,579 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:25:36,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:36,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:36,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:25:36,580 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:25:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:36,583 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993936583"}]},"ts":"1731993936583"} 2024-11-19T05:25:36,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:25:36,584 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:25:36,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:25:36,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, UNASSIGN}] 2024-11-19T05:25:36,587 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, UNASSIGN 2024-11-19T05:25:36,588 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7de0717bd410d1dc7ce54e46b0058ca3, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:36,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, UNASSIGN because future has completed 2024-11-19T05:25:36,590 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:25:36,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:36,742 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:36,742 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:25:36,743 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 7de0717bd410d1dc7ce54e46b0058ca3, disabling compactions & flushes 2024-11-19T05:25:36,743 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:36,743 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:36,743 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. after waiting 0 ms 2024-11-19T05:25:36,743 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:36,743 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 7de0717bd410d1dc7ce54e46b0058ca3 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T05:25:36,760 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0130e7e124d942a39072e8a184eeb57a is 50, key is test_row_1/A:col10/1731993936577/Put/seqid=0 2024-11-19T05:25:36,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742146_1322 (size=9857) 2024-11-19T05:25:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:37,163 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0130e7e124d942a39072e8a184eeb57a 2024-11-19T05:25:37,170 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7abac08611864b19bd6ec1acefcac834 is 50, key is test_row_1/B:col10/1731993936577/Put/seqid=0 2024-11-19T05:25:37,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742147_1323 (size=9857) 2024-11-19T05:25:37,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:37,574 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7abac08611864b19bd6ec1acefcac834 2024-11-19T05:25:37,583 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/61e986b073204c06b35899febdeae382 is 50, key is test_row_1/C:col10/1731993936577/Put/seqid=0 2024-11-19T05:25:37,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742148_1324 (size=9857) 2024-11-19T05:25:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:37,989 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/61e986b073204c06b35899febdeae382 2024-11-19T05:25:37,996 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/A/0130e7e124d942a39072e8a184eeb57a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0130e7e124d942a39072e8a184eeb57a 2024-11-19T05:25:38,000 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0130e7e124d942a39072e8a184eeb57a, entries=100, sequenceid=390, filesize=9.6 K 2024-11-19T05:25:38,001 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/B/7abac08611864b19bd6ec1acefcac834 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7abac08611864b19bd6ec1acefcac834 2024-11-19T05:25:38,005 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7abac08611864b19bd6ec1acefcac834, entries=100, sequenceid=390, filesize=9.6 K 2024-11-19T05:25:38,006 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/.tmp/C/61e986b073204c06b35899febdeae382 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/61e986b073204c06b35899febdeae382 2024-11-19T05:25:38,009 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/61e986b073204c06b35899febdeae382, entries=100, sequenceid=390, filesize=9.6 K 2024-11-19T05:25:38,009 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1266ms, sequenceid=390, compaction requested=false 2024-11-19T05:25:38,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/c6e6ccb35fc74503ad19e30959baaefa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f1d2afb80365475e859174d2cc59c8fc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/fe31a0d5e9a342e3848135968541534c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/901a602aedd748a7b4a50e6b7460055c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/35f00535b80f411791189369b5be9181, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/03b2317ca7f047ce93e2ed01c826e070, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/cde52e7b451b4ded98cf40125193db53, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/aa8e8057c8fb4e3f814c7cb2e402c887, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22] to archive 2024-11-19T05:25:38,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:38,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f16f5a95435649b8b2f4b8d177bac660 2024-11-19T05:25:38,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/89fc1de128c74d598dfe3010dc8506b9 2024-11-19T05:25:38,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/bb9e7f968b1941f2ab1d28b5741cfb75 2024-11-19T05:25:38,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/c6e6ccb35fc74503ad19e30959baaefa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/c6e6ccb35fc74503ad19e30959baaefa 2024-11-19T05:25:38,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/9291823375c948b5bc56e7966878ddbe 2024-11-19T05:25:38,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f1d2afb80365475e859174d2cc59c8fc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/f1d2afb80365475e859174d2cc59c8fc 2024-11-19T05:25:38,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e373fe4709bc40fc8c71da67b5b9d7e9 2024-11-19T05:25:38,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/e59a98950fa94af08d47581b59a29e6b 2024-11-19T05:25:38,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/fe31a0d5e9a342e3848135968541534c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/fe31a0d5e9a342e3848135968541534c 2024-11-19T05:25:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0c24a6d9e59a410dbb7a5ee101b5dbfc 2024-11-19T05:25:38,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/17341a6d002746d9a0366b9e7bc08e64 2024-11-19T05:25:38,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/901a602aedd748a7b4a50e6b7460055c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/901a602aedd748a7b4a50e6b7460055c 2024-11-19T05:25:38,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/30ce08ff9c72463c9015c30a1f1d4586 2024-11-19T05:25:38,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b5c6d0e82df4285ab2c60714a1544fc 2024-11-19T05:25:38,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/35f00535b80f411791189369b5be9181 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/35f00535b80f411791189369b5be9181 2024-11-19T05:25:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/5b8a99b2f14941c88d1c6dd2926e3c07 2024-11-19T05:25:38,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/d8c1bbe2892a4b628f9965d1586a709e 2024-11-19T05:25:38,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/2b7727f677d4459ba0ca784316af779b 2024-11-19T05:25:38,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/03b2317ca7f047ce93e2ed01c826e070 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/03b2317ca7f047ce93e2ed01c826e070 2024-11-19T05:25:38,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/725dd30b28cb4707b7aab41e703b6ddf 2024-11-19T05:25:38,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/cde52e7b451b4ded98cf40125193db53 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/cde52e7b451b4ded98cf40125193db53 2024-11-19T05:25:38,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/766c85f0faa945e7a4ffc0eb5ac658f5 2024-11-19T05:25:38,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/007516f228a94dc09cf998e86008fbc9 2024-11-19T05:25:38,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/a8d5eaffd8e14156a29303a86f31fcdc 2024-11-19T05:25:38,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/aa8e8057c8fb4e3f814c7cb2e402c887 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/aa8e8057c8fb4e3f814c7cb2e402c887 2024-11-19T05:25:38,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/ee67dd917d2143b0aee31e0b698539da 2024-11-19T05:25:38,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/22f1376e93054ff7bae26dad552e2b22 2024-11-19T05:25:38,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/ea7e23e5415342c396dd595707998fde, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/55bfcb4b516546beb0f1a8066a63446c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/bf5bde88bbc0450497cd3725321e92ef, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/0b1faf4173f44eacad28463e972bbc1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/be9033e8415d4aae96a5720479efde7f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/4872674a36784d33b7499f1303fe7e37, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fbd67c7c2ade4e1fa93e81a11176c8db, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fa724a82d1c34e9daa2b56972f920f9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252] to archive 2024-11-19T05:25:38,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:38,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/da448e01c91f4be5bd79448c7632f95c 2024-11-19T05:25:38,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/17c3109d0f654b2ea58d34b9d92232ad 2024-11-19T05:25:38,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/ea7e23e5415342c396dd595707998fde to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/ea7e23e5415342c396dd595707998fde 2024-11-19T05:25:38,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5c82b8de53124db8879031d2d44eb9f1 2024-11-19T05:25:38,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/194ffed3e8724684a053f0b4e9672138 2024-11-19T05:25:38,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/55bfcb4b516546beb0f1a8066a63446c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/55bfcb4b516546beb0f1a8066a63446c 2024-11-19T05:25:38,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/5778955e82744ad99b28e537230c16b1 2024-11-19T05:25:38,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/aec574d8207b40aa835803519b9ea779 2024-11-19T05:25:38,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/bf5bde88bbc0450497cd3725321e92ef to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/bf5bde88bbc0450497cd3725321e92ef 2024-11-19T05:25:38,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8b8d84d7914347ef8806ff3ac6b2d70b 2024-11-19T05:25:38,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/e4dfc6be431c4120ae3894d404838ecd 2024-11-19T05:25:38,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/0b1faf4173f44eacad28463e972bbc1b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/0b1faf4173f44eacad28463e972bbc1b 2024-11-19T05:25:38,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/a864661e6fce4f6cb452845a68638ce7 2024-11-19T05:25:38,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/df30360f22a6443c8ef57390ce45a59b 2024-11-19T05:25:38,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/be9033e8415d4aae96a5720479efde7f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/be9033e8415d4aae96a5720479efde7f 2024-11-19T05:25:38,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/445c63107a1e4b81bc4037d4a7d2a7be 2024-11-19T05:25:38,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/766e2c8267b648a58cf014c19aa6cc79 2024-11-19T05:25:38,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/4872674a36784d33b7499f1303fe7e37 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/4872674a36784d33b7499f1303fe7e37 2024-11-19T05:25:38,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/191e9b61c0b345769e6975eb1a425ef5 2024-11-19T05:25:38,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2da59bc890c5481c9066bf24e21e1fd4 2024-11-19T05:25:38,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fbd67c7c2ade4e1fa93e81a11176c8db to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fbd67c7c2ade4e1fa93e81a11176c8db 2024-11-19T05:25:38,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/2f829522b05940d382027f6ba83c6b71 2024-11-19T05:25:38,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/889ec3a07ab742e29d17a133dbae9aba 2024-11-19T05:25:38,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fa724a82d1c34e9daa2b56972f920f9a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fa724a82d1c34e9daa2b56972f920f9a 2024-11-19T05:25:38,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8a9987ec1b814d91b209c4ad3a0504bc 2024-11-19T05:25:38,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/8aa1eff91de5467b98fe3345da1770b4 2024-11-19T05:25:38,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7d1af9d9b21149a7ba984db7c5c18252 2024-11-19T05:25:38,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/04aefedcc2334d709ca127d5b5e1f878, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e39bd42fb5bf403f85f93668fef1738c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/44b47fa46c9f4e0fbf6d82cf4fb78a06, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/094d409951074ee4b145be4b9065ddb8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/824bc8c77e3641bba7cabc949427dcf0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/024df7cc105047f5a52217068a5363ac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f7062c50f0124a5c90c11eb1138d4b40, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/123c73dfc1b04905be8b66bba8040e6a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323] to archive 2024-11-19T05:25:38,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:25:38,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/21605b7694164b08ba99b99c20b3d6d2 2024-11-19T05:25:38,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/62276e49c2874e70a21f253c1d01bcbc 2024-11-19T05:25:38,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/04aefedcc2334d709ca127d5b5e1f878 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/04aefedcc2334d709ca127d5b5e1f878 2024-11-19T05:25:38,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/5841153f4701410fb625d28ac91eb862 2024-11-19T05:25:38,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/463667c876e34c61a611f88bb2c6cc46 2024-11-19T05:25:38,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e39bd42fb5bf403f85f93668fef1738c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e39bd42fb5bf403f85f93668fef1738c 2024-11-19T05:25:38,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/12e71160f6a54530818d740d9710b9b7 2024-11-19T05:25:38,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/24a3104a24024b1e9703bfbf8a792b11 2024-11-19T05:25:38,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/44b47fa46c9f4e0fbf6d82cf4fb78a06 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/44b47fa46c9f4e0fbf6d82cf4fb78a06 2024-11-19T05:25:38,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f31f2d7b29b442c9996f3ab1dc42e9b7 2024-11-19T05:25:38,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/956bb99e044448eb90f99a6e8c910913 2024-11-19T05:25:38,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/094d409951074ee4b145be4b9065ddb8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/094d409951074ee4b145be4b9065ddb8 2024-11-19T05:25:38,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/60b77b016b794de08d2926027be722f7 2024-11-19T05:25:38,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/b396a59bc2a345d389aa5064be9a4073 2024-11-19T05:25:38,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/824bc8c77e3641bba7cabc949427dcf0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/824bc8c77e3641bba7cabc949427dcf0 2024-11-19T05:25:38,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/8c5baea1a6d741c7af5a49ad128fe4df 2024-11-19T05:25:38,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/1a19db3663af42c2a6aefd4028561414 2024-11-19T05:25:38,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/024df7cc105047f5a52217068a5363ac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/024df7cc105047f5a52217068a5363ac 2024-11-19T05:25:38,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/7623ca93050a49fca871f97b1d577d6f 2024-11-19T05:25:38,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6a05eb6f7c40430d926975422dc61c69 2024-11-19T05:25:38,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f7062c50f0124a5c90c11eb1138d4b40 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/f7062c50f0124a5c90c11eb1138d4b40 2024-11-19T05:25:38,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/55916631dac8415a8bc3fd4809d90f83 2024-11-19T05:25:38,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/29c04d8aa37a4756952c5121a3c50edf 2024-11-19T05:25:38,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/123c73dfc1b04905be8b66bba8040e6a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/123c73dfc1b04905be8b66bba8040e6a 2024-11-19T05:25:38,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/e702bf2c0c9046c49baa88d1f4d4c0cd 2024-11-19T05:25:38,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/6ba64f97379241d7a220e43c8434915d 2024-11-19T05:25:38,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/442d5f27cc434b97b589717de0779323 2024-11-19T05:25:38,089 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/recovered.edits/393.seqid, newMaxSeqId=393, maxSeqId=1 2024-11-19T05:25:38,089 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. 2024-11-19T05:25:38,090 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 7de0717bd410d1dc7ce54e46b0058ca3: Waiting for close lock at 1731993936743Running coprocessor pre-close hooks at 1731993936743Disabling compacts and flushes for region at 1731993936743Disabling writes for close at 1731993936743Obtaining lock to block concurrent updates at 1731993936743Preparing flush snapshotting stores in 7de0717bd410d1dc7ce54e46b0058ca3 at 1731993936743Finished memstore snapshotting TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731993936743Flushing stores of TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3. at 1731993936744 (+1 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/A: creating writer at 1731993936744Flushing 7de0717bd410d1dc7ce54e46b0058ca3/A: appending metadata at 1731993936759 (+15 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/A: closing flushed file at 1731993936759Flushing 7de0717bd410d1dc7ce54e46b0058ca3/B: creating writer at 1731993937167 (+408 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/B: appending metadata at 1731993937169 (+2 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/B: closing flushed file at 1731993937169Flushing 7de0717bd410d1dc7ce54e46b0058ca3/C: creating writer at 1731993937580 (+411 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/C: appending metadata at 1731993937583 (+3 ms)Flushing 7de0717bd410d1dc7ce54e46b0058ca3/C: closing flushed file at 1731993937583Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f015a53: reopening flushed file at 1731993937995 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11f2ca55: reopening flushed file at 1731993938001 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@434df6b7: reopening flushed file at 1731993938005 (+4 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 7de0717bd410d1dc7ce54e46b0058ca3 in 1266ms, sequenceid=390, compaction requested=false at 1731993938009 (+4 ms)Writing region close event to WAL at 1731993938086 (+77 ms)Running coprocessor post-close hooks at 1731993938089 (+3 ms)Closed at 1731993938089 2024-11-19T05:25:38,091 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:38,092 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=7de0717bd410d1dc7ce54e46b0058ca3, regionState=CLOSED 2024-11-19T05:25:38,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:38,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-19T05:25:38,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 7de0717bd410d1dc7ce54e46b0058ca3, server=f990f635ffd0,38449,1731993838792 in 1.5040 sec 2024-11-19T05:25:38,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-19T05:25:38,096 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7de0717bd410d1dc7ce54e46b0058ca3, UNASSIGN in 1.5090 sec 2024-11-19T05:25:38,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-19T05:25:38,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5110 sec 2024-11-19T05:25:38,099 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993938099"}]},"ts":"1731993938099"} 2024-11-19T05:25:38,101 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:25:38,101 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:25:38,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-19T05:25:38,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T05:25:38,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-19T05:25:38,737 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:38,738 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:25:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,739 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T05:25:38,740 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,743 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:38,745 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/recovered.edits] 2024-11-19T05:25:38,749 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0130e7e124d942a39072e8a184eeb57a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/0130e7e124d942a39072e8a184eeb57a 2024-11-19T05:25:38,751 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/4df92a04e90446aa89f08d7343a28558 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/A/4df92a04e90446aa89f08d7343a28558 2024-11-19T05:25:38,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7abac08611864b19bd6ec1acefcac834 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/7abac08611864b19bd6ec1acefcac834 2024-11-19T05:25:38,757 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fdb4403921784188ab81d384daad6ece to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/B/fdb4403921784188ab81d384daad6ece 2024-11-19T05:25:38,759 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/61e986b073204c06b35899febdeae382 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/61e986b073204c06b35899febdeae382 2024-11-19T05:25:38,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/72862c3132f64757a4cdf1db20b01750 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/C/72862c3132f64757a4cdf1db20b01750 2024-11-19T05:25:38,764 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/recovered.edits/393.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3/recovered.edits/393.seqid 2024-11-19T05:25:38,764 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/7de0717bd410d1dc7ce54e46b0058ca3 2024-11-19T05:25:38,764 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:25:38,766 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,769 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:25:38,771 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:25:38,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,772 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:25:38,772 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731993938772"}]},"ts":"9223372036854775807"} 2024-11-19T05:25:38,774 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:25:38,774 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7de0717bd410d1dc7ce54e46b0058ca3, NAME => 'TestAcidGuarantees,,1731993904597.7de0717bd410d1dc7ce54e46b0058ca3.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:25:38,774 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:25:38,774 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731993938774"}]},"ts":"9223372036854775807"} 2024-11-19T05:25:38,776 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:25:38,776 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-19T05:25:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-19T05:25:38,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:25:38,847 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:38,854 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=237 (was 242), OpenFileDescriptor=451 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 395) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 9) - ProcessCount LEAK? -, AvailableMemoryMB=12795 (was 12914) 2024-11-19T05:25:38,861 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=237, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=12795 2024-11-19T05:25:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:25:38,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:25:38,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:38,865 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:25:38,865 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:38,865 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-19T05:25:38,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-19T05:25:38,866 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:25:38,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742149_1325 (size=959) 2024-11-19T05:25:38,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-19T05:25:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-19T05:25:39,276 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:25:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742150_1326 (size=53) 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 54ee91950f7dd8cedd166132c7496a58, disabling compactions & flushes 2024-11-19T05:25:39,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. after waiting 0 ms 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,290 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,290 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 54ee91950f7dd8cedd166132c7496a58: Waiting for close lock at 1731993939290Disabling compacts and flushes for region at 1731993939290Disabling writes for close at 1731993939290Writing region close event to WAL at 1731993939290Closed at 1731993939290 2024-11-19T05:25:39,292 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:25:39,292 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993939292"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993939292"}]},"ts":"1731993939292"} 2024-11-19T05:25:39,294 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:25:39,296 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:25:39,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993939296"}]},"ts":"1731993939296"} 2024-11-19T05:25:39,298 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:25:39,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, ASSIGN}] 2024-11-19T05:25:39,300 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, ASSIGN 2024-11-19T05:25:39,301 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:25:39,452 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:39,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, ASSIGN because future has completed 2024-11-19T05:25:39,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-19T05:25:39,609 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,610 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:25:39,610 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,610 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:25:39,610 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,610 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,612 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,616 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName A 2024-11-19T05:25:39,617 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:39,617 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:39,617 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,619 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName B 2024-11-19T05:25:39,619 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:39,619 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:39,619 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,620 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName C 2024-11-19T05:25:39,620 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:39,621 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:39,621 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,622 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,622 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,623 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,623 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,623 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:25:39,624 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,626 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:25:39,626 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 54ee91950f7dd8cedd166132c7496a58; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68657714, jitterRate=0.023079663515090942}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:25:39,626 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:39,627 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 54ee91950f7dd8cedd166132c7496a58: Running coprocessor pre-open hook at 1731993939610Writing region info on filesystem at 1731993939610Initializing all the Stores at 1731993939611 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993939611Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993939611Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993939611Cleaning up temporary data from old regions at 1731993939623 (+12 ms)Running coprocessor post-open hooks at 1731993939626 (+3 ms)Region opened successfully at 1731993939627 (+1 ms) 2024-11-19T05:25:39,628 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., pid=85, masterSystemTime=1731993939606 2024-11-19T05:25:39,630 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,630 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:39,630 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:39,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:39,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-19T05:25:39,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 in 179 msec 2024-11-19T05:25:39,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-19T05:25:39,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, ASSIGN in 337 msec 2024-11-19T05:25:39,637 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:25:39,637 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993939637"}]},"ts":"1731993939637"} 2024-11-19T05:25:39,639 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:25:39,640 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:25:39,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 777 msec 2024-11-19T05:25:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-19T05:25:39,998 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:39,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:39,999 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:39,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,000 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761dd8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a03b7c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,002 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,005 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:25:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:25:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ba8691a 2024-11-19T05:25:40,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:25:40,006 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:25:40,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:25:40,008 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T05:25:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742151_1327 (size=995) 2024-11-19T05:25:40,419 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T05:25:40,419 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T05:25:40,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:25:40,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE}] 2024-11-19T05:25:40,423 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE 2024-11-19T05:25:40,424 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:40,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE because future has completed 2024-11-19T05:25:40,425 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:25:40,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:40,579 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,579 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:25:40,579 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 54ee91950f7dd8cedd166132c7496a58, disabling compactions & flushes 2024-11-19T05:25:40,579 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,579 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,579 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. after waiting 0 ms 2024-11-19T05:25:40,579 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,583 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T05:25:40,583 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,584 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 54ee91950f7dd8cedd166132c7496a58: Waiting for close lock at 1731993940579Running coprocessor pre-close hooks at 1731993940579Disabling compacts and flushes for region at 1731993940579Disabling writes for close at 1731993940579Writing region close event to WAL at 1731993940579Running coprocessor post-close hooks at 1731993940583 (+4 ms)Closed at 1731993940583 2024-11-19T05:25:40,584 WARN [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 54ee91950f7dd8cedd166132c7496a58 to self. 2024-11-19T05:25:40,586 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,587 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=CLOSED 2024-11-19T05:25:40,588 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:40,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-19T05:25:40,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 in 163 msec 2024-11-19T05:25:40,592 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE; state=CLOSED, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=true 2024-11-19T05:25:40,742 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:40,744 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE because future has completed 2024-11-19T05:25:40,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:25:40,899 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,899 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:25:40,899 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,899 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:25:40,899 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,899 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,900 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,901 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName A 2024-11-19T05:25:40,902 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:40,902 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:40,903 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,903 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName B 2024-11-19T05:25:40,903 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:40,904 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:40,904 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,905 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54ee91950f7dd8cedd166132c7496a58 columnFamilyName C 2024-11-19T05:25:40,905 DEBUG [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:40,905 INFO [StoreOpener-54ee91950f7dd8cedd166132c7496a58-1 {}] regionserver.HStore(327): Store=54ee91950f7dd8cedd166132c7496a58/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:25:40,905 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,906 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,906 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,907 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,907 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,907 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:25:40,909 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,909 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 54ee91950f7dd8cedd166132c7496a58; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69149041, jitterRate=0.03040100634098053}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:25:40,909 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,910 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 54ee91950f7dd8cedd166132c7496a58: Running coprocessor pre-open hook at 1731993940899Writing region info on filesystem at 1731993940899Initializing all the Stores at 1731993940900 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993940900Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993940900Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993940900Cleaning up temporary data from old regions at 1731993940907 (+7 ms)Running coprocessor post-open hooks at 1731993940909 (+2 ms)Region opened successfully at 1731993940910 (+1 ms) 2024-11-19T05:25:40,911 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., pid=90, masterSystemTime=1731993940896 2024-11-19T05:25:40,912 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,912 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:40,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=OPEN, openSeqNum=5, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:40,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:25:40,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-19T05:25:40,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 in 171 msec 2024-11-19T05:25:40,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-19T05:25:40,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, REOPEN/MOVE in 493 msec 2024-11-19T05:25:40,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-19T05:25:40,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-19T05:25:40,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 911 msec 2024-11-19T05:25:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-19T05:25:40,922 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:40,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,923 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6fbf13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,924 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1536f023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,926 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7863cabc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,927 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f3b1931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,929 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4240a60e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b284cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0ad803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,934 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a53bf2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,937 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78bf7424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,939 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3072f660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,944 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,945 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a21a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,947 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a242cd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,949 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,949 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,949 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168582b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,951 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c662118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,952 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e612e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,954 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55324ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616f2ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,958 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444dc603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a84aab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:25:40,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:25:40,960 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:25:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:25:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:25:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1932e7ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:25:40,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:25:40,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:25:40,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:25:40,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175e13ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:25:40,964 DEBUG [Thread-1505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,969 DEBUG [Thread-1514 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,969 DEBUG [Thread-1512 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,971 DEBUG [Thread-1508 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,971 DEBUG [Thread-1503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,972 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,975 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:40,976 DEBUG [Thread-1503 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-19T05:25:40,977 DEBUG [Thread-1503 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:40,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:40,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,981 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:40,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:40,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,992 DEBUG [Thread-1510 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:40,993 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:40,993 DEBUG [Thread-1497 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:40,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:25:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:40,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,995 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:40,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:41,008 DEBUG [Thread-1499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:41,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:41,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:41,030 DEBUG [Thread-1516 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:41,031 DEBUG [Thread-1501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:41,032 DEBUG [Thread-1501 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:25:41,032 DEBUG [Thread-1501 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:25:41,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:41,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:25:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:41,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:25:41,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119330ad4a0bd0f44adaf9cc61fdda0117f_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993940982/Put/seqid=0 2024-11-19T05:25:41,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742152_1328 (size=14594) 2024-11-19T05:25:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:41,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994001051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994001053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994001092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994001092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994001092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994001197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994001199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994001210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994001210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,215 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994001210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:41,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994001407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994001417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994001418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994001427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994001427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,466 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:41,469 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119330ad4a0bd0f44adaf9cc61fdda0117f_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119330ad4a0bd0f44adaf9cc61fdda0117f_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:41,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/44c113e40aef4f81a9e5b3a52a416bd1, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:41,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/44c113e40aef4f81a9e5b3a52a416bd1 is 175, key is test_row_0/A:col10/1731993940982/Put/seqid=0 2024-11-19T05:25:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742153_1329 (size=39549) 2024-11-19T05:25:41,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:41,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994001727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994001738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994001738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994001738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:41,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994001747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:41,756 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:41,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:41,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:41,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:41,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/44c113e40aef4f81a9e5b3a52a416bd1 2024-11-19T05:25:41,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/1c6eb646fb2e4e7f9a1c7c5281e69594 is 50, key is test_row_0/B:col10/1731993940982/Put/seqid=0 2024-11-19T05:25:42,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742154_1330 (size=12001) 2024-11-19T05:25:42,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/1c6eb646fb2e4e7f9a1c7c5281e69594 2024-11-19T05:25:42,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/2921fc95b45d443da378fcb514cdd138 is 50, key is test_row_0/C:col10/1731993940982/Put/seqid=0 2024-11-19T05:25:42,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:42,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742155_1331 (size=12001) 2024-11-19T05:25:42,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:42,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:42,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994002237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994002257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994002257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994002258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994002267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:42,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:42,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:42,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/2921fc95b45d443da378fcb514cdd138 2024-11-19T05:25:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/44c113e40aef4f81a9e5b3a52a416bd1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1 2024-11-19T05:25:42,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1, entries=200, sequenceid=16, filesize=38.6 K 2024-11-19T05:25:42,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/1c6eb646fb2e4e7f9a1c7c5281e69594 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594 2024-11-19T05:25:42,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:25:42,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/2921fc95b45d443da378fcb514cdd138 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138 2024-11-19T05:25:42,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:25:42,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 54ee91950f7dd8cedd166132c7496a58 in 1564ms, sequenceid=16, compaction requested=false 2024-11-19T05:25:42,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:42,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-19T05:25:42,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:42,673 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:25:42,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194f9f428effc84593a7da425be1b04e03_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993941061/Put/seqid=0 2024-11-19T05:25:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742156_1332 (size=12154) 2024-11-19T05:25:43,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:43,126 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194f9f428effc84593a7da425be1b04e03_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f9f428effc84593a7da425be1b04e03_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:43,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/26d3d13225ed4d07be39f59484b18aa7, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:43,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/26d3d13225ed4d07be39f59484b18aa7 is 175, key is test_row_0/A:col10/1731993941061/Put/seqid=0 2024-11-19T05:25:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742157_1333 (size=30955) 2024-11-19T05:25:43,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:43,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:43,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994003297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994003298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994003301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994003309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994003310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994003417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994003417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994003437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994003437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994003427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,581 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/26d3d13225ed4d07be39f59484b18aa7 2024-11-19T05:25:43,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/d491e30e12984db9a5733306568d63a1 is 50, key is test_row_0/B:col10/1731993941061/Put/seqid=0 2024-11-19T05:25:43,636 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:25:43,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994003637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994003637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742158_1334 (size=12001) 2024-11-19T05:25:43,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994003658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994003658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994003659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994003960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994003961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994003970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994003977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:43,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994003977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/d491e30e12984db9a5733306568d63a1 2024-11-19T05:25:44,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/d7ff021e06084a72b4a722caa0db44cb is 50, key is test_row_0/C:col10/1731993941061/Put/seqid=0 2024-11-19T05:25:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742159_1335 (size=12001) 2024-11-19T05:25:44,116 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/d7ff021e06084a72b4a722caa0db44cb 2024-11-19T05:25:44,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/26d3d13225ed4d07be39f59484b18aa7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7 2024-11-19T05:25:44,125 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7, entries=150, sequenceid=40, filesize=30.2 K 2024-11-19T05:25:44,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/d491e30e12984db9a5733306568d63a1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1 2024-11-19T05:25:44,130 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T05:25:44,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/d7ff021e06084a72b4a722caa0db44cb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb 2024-11-19T05:25:44,135 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb, entries=150, sequenceid=40, filesize=11.7 K 2024-11-19T05:25:44,136 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 54ee91950f7dd8cedd166132c7496a58 in 1463ms, sequenceid=40, compaction requested=false 2024-11-19T05:25:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:44,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:44,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-19T05:25:44,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-19T05:25:44,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-19T05:25:44,140 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1570 sec 2024-11-19T05:25:44,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 3.1660 sec 2024-11-19T05:25:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:44,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:44,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:25:44,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195cf7bbaaf5d84eb9ab7a843ee195deb7_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742160_1336 (size=14594) 2024-11-19T05:25:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994004539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994004539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994004543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994004545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994004548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994004657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994004657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994004667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994004667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994004668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994004877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994004877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994004888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994004888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994004888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:44,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:44,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:44,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195cf7bbaaf5d84eb9ab7a843ee195deb7_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cf7bbaaf5d84eb9ab7a843ee195deb7_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:44,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/109081d8ff5b48638209e915e41d4fbe, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:44,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/109081d8ff5b48638209e915e41d4fbe is 175, key is test_row_0/A:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:44,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742161_1337 (size=39549) 2024-11-19T05:25:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-19T05:25:45,148 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:45,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-19T05:25:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:45,165 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:45,166 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:45,166 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:45,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994005198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994005198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994005207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994005207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994005208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:45,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T05:25:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,367 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/109081d8ff5b48638209e915e41d4fbe 2024-11-19T05:25:45,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/fee2082fbb6145238c93a27295e9afc1 is 50, key is test_row_0/B:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:45,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742162_1338 (size=12001) 2024-11-19T05:25:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:45,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T05:25:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:45,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T05:25:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994005717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994005717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994005727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994005727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994005738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:45,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/fee2082fbb6145238c93a27295e9afc1 2024-11-19T05:25:45,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T05:25:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:45,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 is 50, key is test_row_0/C:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:45,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742163_1339 (size=12001) 2024-11-19T05:25:45,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 2024-11-19T05:25:45,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/109081d8ff5b48638209e915e41d4fbe as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe 2024-11-19T05:25:45,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe, entries=200, sequenceid=54, filesize=38.6 K 2024-11-19T05:25:45,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/fee2082fbb6145238c93a27295e9afc1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1 2024-11-19T05:25:45,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T05:25:45,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 2024-11-19T05:25:45,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T05:25:45,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 54ee91950f7dd8cedd166132c7496a58 in 1372ms, sequenceid=54, compaction requested=true 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:45,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T05:25:45,856 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:45,856 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:45,857 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:45,857 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:25:45,857 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,857 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.2 K 2024-11-19T05:25:45,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:45,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:25:45,857 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,857 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=107.5 K 2024-11-19T05:25:45,857 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,857 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe] 2024-11-19T05:25:45,858 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2921fc95b45d443da378fcb514cdd138, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993940982 2024-11-19T05:25:45,858 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44c113e40aef4f81a9e5b3a52a416bd1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993940982 2024-11-19T05:25:45,859 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d3d13225ed4d07be39f59484b18aa7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731993941051 2024-11-19T05:25:45,859 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d7ff021e06084a72b4a722caa0db44cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731993941051 2024-11-19T05:25:45,859 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109081d8ff5b48638209e915e41d4fbe, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:45,859 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 27e6c1f821ee46c1aca0f6c6f6d9dac8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:45,871 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:45,872 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/234473eb417f4406864938a54558e61a is 50, key is test_row_0/C:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:45,881 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:45,889 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119c390dc0725364440b08a48c62c0fc869_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:45,892 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119c390dc0725364440b08a48c62c0fc869_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:45,892 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c390dc0725364440b08a48c62c0fc869_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:45,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742164_1340 (size=12104) 2024-11-19T05:25:45,913 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/234473eb417f4406864938a54558e61a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/234473eb417f4406864938a54558e61a 2024-11-19T05:25:45,923 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into 234473eb417f4406864938a54558e61a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:45,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:45,923 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=13, startTime=1731993945855; duration=0sec 2024-11-19T05:25:45,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:45,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:25:45,923 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:45,925 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:45,925 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:25:45,925 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,925 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.2 K 2024-11-19T05:25:45,925 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c6eb646fb2e4e7f9a1c7c5281e69594, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993940982 2024-11-19T05:25:45,926 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d491e30e12984db9a5733306568d63a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731993941051 2024-11-19T05:25:45,926 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fee2082fbb6145238c93a27295e9afc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:45,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742165_1341 (size=4469) 2024-11-19T05:25:45,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-19T05:25:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:45,952 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:25:45,953 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:45,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/88061b3c92754f2fa9a7d6dde52738e7 is 50, key is test_row_0/B:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198e4d970c3a6942c4b86986d1d3cc10f8_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993944543/Put/seqid=0 2024-11-19T05:25:45,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742166_1342 (size=12104) 2024-11-19T05:25:46,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742167_1343 (size=12154) 2024-11-19T05:25:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:46,343 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#289 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:46,343 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/d91bfbcc9ea548babe2c08dab58a7565 is 175, key is test_row_0/A:col10/1731993943308/Put/seqid=0 2024-11-19T05:25:46,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742168_1344 (size=31058) 2024-11-19T05:25:46,388 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/d91bfbcc9ea548babe2c08dab58a7565 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565 2024-11-19T05:25:46,395 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into d91bfbcc9ea548babe2c08dab58a7565(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:46,395 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:46,395 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=13, startTime=1731993945855; duration=0sec 2024-11-19T05:25:46,395 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:46,395 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:25:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:46,410 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/88061b3c92754f2fa9a7d6dde52738e7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/88061b3c92754f2fa9a7d6dde52738e7 2024-11-19T05:25:46,423 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198e4d970c3a6942c4b86986d1d3cc10f8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e4d970c3a6942c4b86986d1d3cc10f8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:46,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/fa99ea23b4da47b689969f1477c570a8, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:46,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/fa99ea23b4da47b689969f1477c570a8 is 175, key is test_row_0/A:col10/1731993944543/Put/seqid=0 2024-11-19T05:25:46,426 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into 88061b3c92754f2fa9a7d6dde52738e7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:46,426 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:46,426 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=13, startTime=1731993945855; duration=0sec 2024-11-19T05:25:46,426 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:46,426 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:25:46,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742169_1345 (size=30955) 2024-11-19T05:25:46,440 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/fa99ea23b4da47b689969f1477c570a8 2024-11-19T05:25:46,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/6032bf113cf04bb1ae576770d9306e98 is 50, key is test_row_0/B:col10/1731993944543/Put/seqid=0 2024-11-19T05:25:46,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742170_1346 (size=12001) 2024-11-19T05:25:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:46,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:46,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994006768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994006782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994006782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994006784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994006785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,870 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/6032bf113cf04bb1ae576770d9306e98 2024-11-19T05:25:46,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/67412190c70a4a6cb2a67529d0e87e67 is 50, key is test_row_0/C:col10/1731993944543/Put/seqid=0 2024-11-19T05:25:46,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742171_1347 (size=12001) 2024-11-19T05:25:46,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994006887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994006897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994006907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994006907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:46,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994006907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994007107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994007107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994007117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994007127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994007128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,290 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/67412190c70a4a6cb2a67529d0e87e67 2024-11-19T05:25:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/fa99ea23b4da47b689969f1477c570a8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8 2024-11-19T05:25:47,298 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8, entries=150, sequenceid=77, filesize=30.2 K 2024-11-19T05:25:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/6032bf113cf04bb1ae576770d9306e98 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98 2024-11-19T05:25:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:47,313 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:25:47,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/67412190c70a4a6cb2a67529d0e87e67 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67 2024-11-19T05:25:47,324 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67, entries=150, sequenceid=77, filesize=11.7 K 2024-11-19T05:25:47,330 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 54ee91950f7dd8cedd166132c7496a58 in 1379ms, sequenceid=77, compaction requested=false 2024-11-19T05:25:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:47,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-19T05:25:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-19T05:25:47,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-19T05:25:47,336 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-19T05:25:47,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.1890 sec 2024-11-19T05:25:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:47,435 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:47,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:25:47,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f57ee0b022124dd59404c2bbc1752de8_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:47,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742172_1348 (size=14594) 2024-11-19T05:25:47,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:47,463 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119f57ee0b022124dd59404c2bbc1752de8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f57ee0b022124dd59404c2bbc1752de8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:47,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/c69cb65328264033b844f09205995cae, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:47,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/c69cb65328264033b844f09205995cae is 175, key is test_row_0/A:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:47,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994007475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994007482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994007482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994007487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994007487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742173_1349 (size=39549) 2024-11-19T05:25:47,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994007587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994007607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994007608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994007608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994007608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,622 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994007807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994007817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994007827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994007827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994007827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:47,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/c69cb65328264033b844f09205995cae 2024-11-19T05:25:47,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c4b4fed21f374bdb99a819d7280bc9a0 is 50, key is test_row_0/B:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:47,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742174_1350 (size=12001) 2024-11-19T05:25:47,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c4b4fed21f374bdb99a819d7280bc9a0 2024-11-19T05:25:47,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/14830721f6e3447a93055f2bd19f65a3 is 50, key is test_row_0/C:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:47,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742175_1351 (size=12001) 2024-11-19T05:25:47,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/14830721f6e3447a93055f2bd19f65a3 2024-11-19T05:25:47,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/c69cb65328264033b844f09205995cae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae 2024-11-19T05:25:47,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae, entries=200, sequenceid=95, filesize=38.6 K 2024-11-19T05:25:47,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c4b4fed21f374bdb99a819d7280bc9a0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0 2024-11-19T05:25:47,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T05:25:47,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/14830721f6e3447a93055f2bd19f65a3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3 2024-11-19T05:25:47,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T05:25:47,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 54ee91950f7dd8cedd166132c7496a58 in 552ms, sequenceid=95, compaction requested=true 2024-11-19T05:25:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:47,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:47,988 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:47,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:47,991 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:47,991 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:25:47,991 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:47,991 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=99.2 K 2024-11-19T05:25:47,991 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:47,992 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae] 2024-11-19T05:25:47,992 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting d91bfbcc9ea548babe2c08dab58a7565, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:47,993 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa99ea23b4da47b689969f1477c570a8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993944515 2024-11-19T05:25:47,996 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69cb65328264033b844f09205995cae, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946778 2024-11-19T05:25:48,000 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:48,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:48,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:48,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:48,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:48,001 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:48,001 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:25:48,001 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:48,001 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/88061b3c92754f2fa9a7d6dde52738e7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.3 K 2024-11-19T05:25:48,001 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 88061b3c92754f2fa9a7d6dde52738e7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:48,001 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6032bf113cf04bb1ae576770d9306e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993944515 2024-11-19T05:25:48,002 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c4b4fed21f374bdb99a819d7280bc9a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946782 2024-11-19T05:25:48,025 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:48,040 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:48,040 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/ffdab3e48db7448b8f12b1e1e2ee4eca is 50, key is test_row_0/B:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:48,044 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195f7a9ad3e2f6472ead2cd15f7d1585e9_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:48,047 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195f7a9ad3e2f6472ead2cd15f7d1585e9_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:48,047 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195f7a9ad3e2f6472ead2cd15f7d1585e9_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:48,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742176_1352 (size=12207) 2024-11-19T05:25:48,096 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/ffdab3e48db7448b8f12b1e1e2ee4eca as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/ffdab3e48db7448b8f12b1e1e2ee4eca 2024-11-19T05:25:48,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742177_1353 (size=4469) 2024-11-19T05:25:48,108 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into ffdab3e48db7448b8f12b1e1e2ee4eca(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:48,108 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:48,108 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=13, startTime=1731993947990; duration=0sec 2024-11-19T05:25:48,108 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:48,118 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:25:48,118 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:48,119 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:48,119 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:25:48,119 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:48,119 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/234473eb417f4406864938a54558e61a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.3 K 2024-11-19T05:25:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:48,121 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:25:48,122 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 234473eb417f4406864938a54558e61a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993943295 2024-11-19T05:25:48,128 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 67412190c70a4a6cb2a67529d0e87e67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731993944515 2024-11-19T05:25:48,129 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 14830721f6e3447a93055f2bd19f65a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946782 2024-11-19T05:25:48,169 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:48,169 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/826e0bd0812143578ff2f844d30b43b6 is 50, key is test_row_0/C:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:48,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119004bde8d274e4c16bbd4ad4bc8a7dfd8_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993948120/Put/seqid=0 2024-11-19T05:25:48,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994008172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994008175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994008177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994008179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994008183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742178_1354 (size=12207) 2024-11-19T05:25:48,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742179_1355 (size=14594) 2024-11-19T05:25:48,255 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:48,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119004bde8d274e4c16bbd4ad4bc8a7dfd8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119004bde8d274e4c16bbd4ad4bc8a7dfd8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:48,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/f2e012429ef84b84af664645690b68be, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:48,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/f2e012429ef84b84af664645690b68be is 175, key is test_row_0/A:col10/1731993948120/Put/seqid=0 2024-11-19T05:25:48,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994008297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742180_1356 (size=39549) 2024-11-19T05:25:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994008297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,314 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/f2e012429ef84b84af664645690b68be 2024-11-19T05:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994008308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994008308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994008308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b2ea7912af7d4001b2f4b24edd054cee is 50, key is test_row_0/B:col10/1731993948120/Put/seqid=0 2024-11-19T05:25:48,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T05:25:48,366 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-19T05:25:48,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742181_1357 (size=12001) 2024-11-19T05:25:48,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b2ea7912af7d4001b2f4b24edd054cee 2024-11-19T05:25:48,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/3a46f248f94f4c9e930ab4cbd05cf8cb is 50, key is test_row_0/C:col10/1731993948120/Put/seqid=0 2024-11-19T05:25:48,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742182_1358 (size=12001) 2024-11-19T05:25:48,500 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#297 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:48,500 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7297594f868c4549aa6f713f53abf85c is 175, key is test_row_0/A:col10/1731993946782/Put/seqid=0 2024-11-19T05:25:48,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742183_1359 (size=31161) 2024-11-19T05:25:48,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994008517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994008517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994008527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994008527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994008538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,652 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/826e0bd0812143578ff2f844d30b43b6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/826e0bd0812143578ff2f844d30b43b6 2024-11-19T05:25:48,660 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into 826e0bd0812143578ff2f844d30b43b6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:48,660 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:48,660 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=13, startTime=1731993948000; duration=0sec 2024-11-19T05:25:48,660 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:48,660 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:25:48,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994008837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994008837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/3a46f248f94f4c9e930ab4cbd05cf8cb 2024-11-19T05:25:48,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/f2e012429ef84b84af664645690b68be as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be 2024-11-19T05:25:48,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994008847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994008848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be, entries=200, sequenceid=118, filesize=38.6 K 2024-11-19T05:25:48,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b2ea7912af7d4001b2f4b24edd054cee as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee 2024-11-19T05:25:48,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T05:25:48,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:48,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994008857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:48,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:48,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:48,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/3a46f248f94f4c9e930ab4cbd05cf8cb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb 2024-11-19T05:25:48,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-19T05:25:48,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 54ee91950f7dd8cedd166132c7496a58 in 753ms, sequenceid=118, compaction requested=false 2024-11-19T05:25:48,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:48,930 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7297594f868c4549aa6f713f53abf85c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c 2024-11-19T05:25:48,936 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into 7297594f868c4549aa6f713f53abf85c(size=30.4 K), total size for store is 69.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:48,937 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:48,937 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=13, startTime=1731993947987; duration=0sec 2024-11-19T05:25:48,937 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:48,937 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:25:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-19T05:25:49,327 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:49,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-19T05:25:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:49,330 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:49,337 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:49,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:49,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:25:49,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c8635f4c0695431b9b798f283bac272a_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742184_1360 (size=14694) 2024-11-19T05:25:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994009404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994009406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,419 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994009416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994009417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994009418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:49,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994009528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994009528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994009537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994009537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994009537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:49,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:49,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994009739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994009740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994009747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994009758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994009758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:49,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:49,793 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:49,798 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c8635f4c0695431b9b798f283bac272a_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c8635f4c0695431b9b798f283bac272a_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:49,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,799 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/e4d2bfec313f43dd996878f28c795256, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:49,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/e4d2bfec313f43dd996878f28c795256 is 175, key is test_row_0/A:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742185_1361 (size=39649) 2024-11-19T05:25:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:49,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:49,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994010057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994010057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994010067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994010077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994010077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/e4d2bfec313f43dd996878f28c795256 2024-11-19T05:25:50,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b74d3d517203412bb99ee80d2873166e is 50, key is test_row_0/B:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:50,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742186_1362 (size=12101) 2024-11-19T05:25:50,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:50,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994010579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994010579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994010579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994010597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994010597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:50,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:50,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b74d3d517203412bb99ee80d2873166e 2024-11-19T05:25:50,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/72526761d5e54a429005e9f98acc2bf4 is 50, key is test_row_0/C:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742187_1363 (size=12101) 2024-11-19T05:25:50,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:50,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:50,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:51,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:51,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:51,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/72526761d5e54a429005e9f98acc2bf4 2024-11-19T05:25:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/e4d2bfec313f43dd996878f28c795256 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256 2024-11-19T05:25:51,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256, entries=200, sequenceid=135, filesize=38.7 K 2024-11-19T05:25:51,096 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-19T05:25:51,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/b74d3d517203412bb99ee80d2873166e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e 2024-11-19T05:25:51,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e, entries=150, sequenceid=135, filesize=11.8 K 2024-11-19T05:25:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/72526761d5e54a429005e9f98acc2bf4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4 2024-11-19T05:25:51,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-19T05:25:51,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 54ee91950f7dd8cedd166132c7496a58 in 1741ms, sequenceid=135, compaction requested=true 2024-11-19T05:25:51,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:51,108 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:51,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:51,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:51,110 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:51,110 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:25:51,110 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,110 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=107.8 K 2024-11-19T05:25:51,110 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,110 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256] 2024-11-19T05:25:51,110 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:51,110 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7297594f868c4549aa6f713f53abf85c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946782 2024-11-19T05:25:51,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:51,111 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2e012429ef84b84af664645690b68be, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731993947479 2024-11-19T05:25:51,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:51,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:51,111 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4d2bfec313f43dd996878f28c795256, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:51,111 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:51,111 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:25:51,111 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,111 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/ffdab3e48db7448b8f12b1e1e2ee4eca, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.5 K 2024-11-19T05:25:51,112 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdab3e48db7448b8f12b1e1e2ee4eca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946782 2024-11-19T05:25:51,112 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b2ea7912af7d4001b2f4b24edd054cee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731993947484 2024-11-19T05:25:51,112 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b74d3d517203412bb99ee80d2873166e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:51,125 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:51,130 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:51,131 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/27fefea31fa34d05b3087208cad50567 is 50, key is test_row_0/B:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:51,133 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111993961b6187394167b45779d541c4d601_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:51,135 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111993961b6187394167b45779d541c4d601_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:51,136 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111993961b6187394167b45779d541c4d601_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:51,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742188_1364 (size=12409) 2024-11-19T05:25:51,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-19T05:25:51,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,184 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T05:25:51,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742189_1365 (size=4469) 2024-11-19T05:25:51,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111974c0f8c9c94e4851999c8070d9c598e0_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993949402/Put/seqid=0 2024-11-19T05:25:51,196 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#306 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:51,197 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/2e308d5923d74fcf994ed2c60c726851 is 175, key is test_row_0/A:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:51,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742190_1366 (size=12304) 2024-11-19T05:25:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742191_1367 (size=31363) 2024-11-19T05:25:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:51,563 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/27fefea31fa34d05b3087208cad50567 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/27fefea31fa34d05b3087208cad50567 2024-11-19T05:25:51,571 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into 27fefea31fa34d05b3087208cad50567(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:51,572 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:51,572 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=13, startTime=1731993951110; duration=0sec 2024-11-19T05:25:51,572 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:51,572 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:25:51,572 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:51,573 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:51,573 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:25:51,573 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:51,573 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/826e0bd0812143578ff2f844d30b43b6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.5 K 2024-11-19T05:25:51,574 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 826e0bd0812143578ff2f844d30b43b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993946782 2024-11-19T05:25:51,574 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a46f248f94f4c9e930ab4cbd05cf8cb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731993947484 2024-11-19T05:25:51,574 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 72526761d5e54a429005e9f98acc2bf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:51,581 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:51,582 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/0b1d297478674ec59bb2ee7782a6f8dd is 50, key is test_row_0/C:col10/1731993948156/Put/seqid=0 2024-11-19T05:25:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:51,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:51,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742192_1368 (size=12409) 2024-11-19T05:25:51,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:51,615 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111974c0f8c9c94e4851999c8070d9c598e0_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974c0f8c9c94e4851999c8070d9c598e0_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:51,616 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/0b1d297478674ec59bb2ee7782a6f8dd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/0b1d297478674ec59bb2ee7782a6f8dd 2024-11-19T05:25:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/ec5c3b23d3814edcb85580fa9edbda62, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:51,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/ec5c3b23d3814edcb85580fa9edbda62 is 175, key is test_row_0/A:col10/1731993949402/Put/seqid=0 2024-11-19T05:25:51,627 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into 0b1d297478674ec59bb2ee7782a6f8dd(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:51,627 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:51,627 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=13, startTime=1731993951111; duration=0sec 2024-11-19T05:25:51,627 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:51,627 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:25:51,632 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/2e308d5923d74fcf994ed2c60c726851 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851 2024-11-19T05:25:51,639 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into 2e308d5923d74fcf994ed2c60c726851(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:51,639 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:51,639 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=13, startTime=1731993951108; duration=0sec 2024-11-19T05:25:51,639 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:51,639 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:25:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994011632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994011633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742193_1369 (size=31105) 2024-11-19T05:25:51,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/ec5c3b23d3814edcb85580fa9edbda62 2024-11-19T05:25:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994011638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994011639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994011644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/e26255af360e4aa480e9be06ad4088f9 is 50, key is test_row_0/B:col10/1731993949402/Put/seqid=0 2024-11-19T05:25:51,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742194_1370 (size=12151) 2024-11-19T05:25:51,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994011747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994011748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994011757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994011757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994011767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994011968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994011968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994011968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994011977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:51,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:51,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994011987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,063 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/e26255af360e4aa480e9be06ad4088f9 2024-11-19T05:25:52,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/cf2d24781e1840f7bd5fba9a6f861371 is 50, key is test_row_0/C:col10/1731993949402/Put/seqid=0 2024-11-19T05:25:52,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742195_1371 (size=12151) 2024-11-19T05:25:52,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994012277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994012287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994012288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994012297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994012297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,482 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/cf2d24781e1840f7bd5fba9a6f861371 2024-11-19T05:25:52,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/ec5c3b23d3814edcb85580fa9edbda62 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62 2024-11-19T05:25:52,488 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62, entries=150, sequenceid=156, filesize=30.4 K 2024-11-19T05:25:52,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/e26255af360e4aa480e9be06ad4088f9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9 2024-11-19T05:25:52,491 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:25:52,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/cf2d24781e1840f7bd5fba9a6f861371 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371 2024-11-19T05:25:52,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371, entries=150, sequenceid=156, filesize=11.9 K 2024-11-19T05:25:52,496 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 54ee91950f7dd8cedd166132c7496a58 in 1313ms, sequenceid=156, compaction requested=false 2024-11-19T05:25:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:52,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-19T05:25:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-19T05:25:52,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-19T05:25:52,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1600 sec 2024-11-19T05:25:52,500 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.1710 sec 2024-11-19T05:25:52,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:52,791 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:52,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:25:52,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9770c1b4c894239bc2511156404cc08_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:52,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742196_1372 (size=14794) 2024-11-19T05:25:52,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994012824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994012825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994012826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994012826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994012829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994012937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994012937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994012937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994012947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:52,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994012947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994013147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994013147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,154 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994013147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994013157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994013157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,203 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:53,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9770c1b4c894239bc2511156404cc08_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9770c1b4c894239bc2511156404cc08_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:53,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/8520fbbd80744769802d017f4b05c25c, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:53,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/8520fbbd80744769802d017f4b05c25c is 175, key is test_row_0/A:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:53,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742197_1373 (size=39749) 2024-11-19T05:25:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994013457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994013457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994013467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994013477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994013478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-19T05:25:53,487 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:53,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-19T05:25:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:53,489 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:53,490 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:53,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:53,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:53,610 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/8520fbbd80744769802d017f4b05c25c 2024-11-19T05:25:53,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c2152dccd6b34688b357cb2436b8342b is 50, key is test_row_0/B:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:53,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742198_1374 (size=12151) 2024-11-19T05:25:53,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:53,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:53,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:53,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:53,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994013967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994013967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994013977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994013987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994013987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:54,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c2152dccd6b34688b357cb2436b8342b 2024-11-19T05:25:54,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/c7d5b3615e0a498e84050d7b6555f4ee is 50, key is test_row_0/C:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:54,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742199_1375 (size=12151) 2024-11-19T05:25:54,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:54,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:54,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:54,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:54,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:25:54,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/c7d5b3615e0a498e84050d7b6555f4ee 2024-11-19T05:25:54,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/8520fbbd80744769802d017f4b05c25c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c 2024-11-19T05:25:54,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c, entries=200, sequenceid=175, filesize=38.8 K 2024-11-19T05:25:54,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/c2152dccd6b34688b357cb2436b8342b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b 2024-11-19T05:25:54,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T05:25:54,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/c7d5b3615e0a498e84050d7b6555f4ee as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee 2024-11-19T05:25:54,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee, entries=150, sequenceid=175, filesize=11.9 K 2024-11-19T05:25:54,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 54ee91950f7dd8cedd166132c7496a58 in 1653ms, sequenceid=175, compaction requested=true 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:54,444 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:54,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:25:54,445 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:25:54,446 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,446 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=99.8 K 2024-11-19T05:25:54,446 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c] 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:25:54,446 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,446 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e308d5923d74fcf994ed2c60c726851, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:54,447 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/27fefea31fa34d05b3087208cad50567, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.9 K 2024-11-19T05:25:54,447 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec5c3b23d3814edcb85580fa9edbda62, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993949402 2024-11-19T05:25:54,447 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 27fefea31fa34d05b3087208cad50567, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:54,447 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8520fbbd80744769802d017f4b05c25c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:54,447 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e26255af360e4aa480e9be06ad4088f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993949402 2024-11-19T05:25:54,447 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c2152dccd6b34688b357cb2436b8342b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:54,453 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:54,454 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:54,454 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/a2abafdf6f274c4bbcd483ddf87f1787 is 50, key is test_row_0/B:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:54,456 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119b3019fb2f97f4b17bf340487f79ccf41_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:54,458 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119b3019fb2f97f4b17bf340487f79ccf41_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:54,458 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b3019fb2f97f4b17bf340487f79ccf41_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:54,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742200_1376 (size=12561) 2024-11-19T05:25:54,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742201_1377 (size=4469) 2024-11-19T05:25:54,470 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/a2abafdf6f274c4bbcd483ddf87f1787 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/a2abafdf6f274c4bbcd483ddf87f1787 2024-11-19T05:25:54,474 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into a2abafdf6f274c4bbcd483ddf87f1787(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:54,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:54,474 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=13, startTime=1731993954444; duration=0sec 2024-11-19T05:25:54,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:54,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:25:54,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:54,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:54,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:25:54,475 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,475 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/0b1d297478674ec59bb2ee7782a6f8dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=35.9 K 2024-11-19T05:25:54,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b1d297478674ec59bb2ee7782a6f8dd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731993948156 2024-11-19T05:25:54,476 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting cf2d24781e1840f7bd5fba9a6f861371, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731993949402 2024-11-19T05:25:54,476 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d5b3615e0a498e84050d7b6555f4ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:54,481 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:54,482 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/f1f8fbcc2e5b4521bd1a8ae1ab433010 is 50, key is test_row_0/C:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:54,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742202_1378 (size=12561) 2024-11-19T05:25:54,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-19T05:25:54,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:54,558 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:25:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193049ff6fa6f14da2a4d221e049702567_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993952823/Put/seqid=0 2024-11-19T05:25:54,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742203_1379 (size=12304) 2024-11-19T05:25:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:54,869 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:54,869 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3e4ebb46eb8247a68a598a48f789f8e8 is 175, key is test_row_0/A:col10/1731993952790/Put/seqid=0 2024-11-19T05:25:54,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742204_1380 (size=31515) 2024-11-19T05:25:54,888 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/f1f8fbcc2e5b4521bd1a8ae1ab433010 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/f1f8fbcc2e5b4521bd1a8ae1ab433010 2024-11-19T05:25:54,892 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into f1f8fbcc2e5b4521bd1a8ae1ab433010(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:54,892 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:54,892 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=13, startTime=1731993954444; duration=0sec 2024-11-19T05:25:54,892 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:54,892 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:25:54,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:54,971 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193049ff6fa6f14da2a4d221e049702567_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193049ff6fa6f14da2a4d221e049702567_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3683235bfd344384b851b36cb091ebe4, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3683235bfd344384b851b36cb091ebe4 is 175, key is test_row_0/A:col10/1731993952823/Put/seqid=0 2024-11-19T05:25:54,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742205_1381 (size=31105) 2024-11-19T05:25:54,978 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3683235bfd344384b851b36cb091ebe4 2024-11-19T05:25:54,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/f3bfe6bf86c94d45a27b34a742839fd1 is 50, key is test_row_0/B:col10/1731993952823/Put/seqid=0 2024-11-19T05:25:54,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:54,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:55,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742206_1382 (size=12151) 2024-11-19T05:25:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994015016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994015017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994015018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994015019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994015025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994015127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994015127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994015137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,144 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,276 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3e4ebb46eb8247a68a598a48f789f8e8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8 2024-11-19T05:25:55,284 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into 3e4ebb46eb8247a68a598a48f789f8e8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:55,284 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:55,284 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=13, startTime=1731993954444; duration=0sec 2024-11-19T05:25:55,285 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:55,285 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:25:55,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994015337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994015337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994015347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994015347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994015347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,352 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,401 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/f3bfe6bf86c94d45a27b34a742839fd1 2024-11-19T05:25:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a5d4e131c7394513a72831abd4aaaffd is 50, key is test_row_0/C:col10/1731993952823/Put/seqid=0 2024-11-19T05:25:55,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742207_1383 (size=12151) 2024-11-19T05:25:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994015647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994015647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994015657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994015657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:55,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994015657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:55,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:55,811 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a5d4e131c7394513a72831abd4aaaffd 2024-11-19T05:25:55,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3683235bfd344384b851b36cb091ebe4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4 2024-11-19T05:25:55,818 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4, entries=150, sequenceid=196, filesize=30.4 K 2024-11-19T05:25:55,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/f3bfe6bf86c94d45a27b34a742839fd1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1 2024-11-19T05:25:55,822 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T05:25:55,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a5d4e131c7394513a72831abd4aaaffd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd 2024-11-19T05:25:55,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd, entries=150, sequenceid=196, filesize=11.9 K 2024-11-19T05:25:55,827 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 54ee91950f7dd8cedd166132c7496a58 in 1268ms, sequenceid=196, compaction requested=false 2024-11-19T05:25:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-19T05:25:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-19T05:25:55,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-19T05:25:55,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-19T05:25:55,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.3430 sec 2024-11-19T05:25:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:56,160 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:56,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T05:25:56,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c4be75ee893b418bb50233834be98f50_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:56,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742208_1384 (size=14794) 2024-11-19T05:25:56,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994016191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994016192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994016198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994016199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994016199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994016307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994016317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994016317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994016517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994016517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994016517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994016537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994016537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,571 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:56,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119c4be75ee893b418bb50233834be98f50_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4be75ee893b418bb50233834be98f50_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:56,575 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/066f65af78134ae69f85ab8d7c2e23f5, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:56,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/066f65af78134ae69f85ab8d7c2e23f5 is 175, key is test_row_0/A:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:56,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742209_1385 (size=39749) 2024-11-19T05:25:56,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994016826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994016837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994016837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994016857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994016857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:56,980 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/066f65af78134ae69f85ab8d7c2e23f5 2024-11-19T05:25:56,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/9e9f3648671a42f58eb1e98341418831 is 50, key is test_row_0/B:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:56,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742210_1386 (size=12151) 2024-11-19T05:25:57,116 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:25:57,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994017347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994017357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:57,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994017357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:57,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994017367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:57,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994017377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:57,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:57,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/9e9f3648671a42f58eb1e98341418831 2024-11-19T05:25:57,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/96cc5268cc53450cbeb983725dffd9af is 50, key is test_row_0/C:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:57,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742211_1387 (size=12151) 2024-11-19T05:25:57,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/96cc5268cc53450cbeb983725dffd9af 2024-11-19T05:25:57,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/066f65af78134ae69f85ab8d7c2e23f5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5 2024-11-19T05:25:57,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5, entries=200, sequenceid=215, filesize=38.8 K 2024-11-19T05:25:57,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/9e9f3648671a42f58eb1e98341418831 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831 2024-11-19T05:25:57,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T05:25:57,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/96cc5268cc53450cbeb983725dffd9af as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af 2024-11-19T05:25:57,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T05:25:57,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 54ee91950f7dd8cedd166132c7496a58 in 1268ms, sequenceid=215, compaction requested=true 2024-11-19T05:25:57,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:57,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:25:57,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:57,428 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:57,428 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:57,429 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:57,429 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:25:57,429 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:57,429 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=100.0 K 2024-11-19T05:25:57,429 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:57,429 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5] 2024-11-19T05:25:57,429 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:57,429 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:25:57,429 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:57,429 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/a2abafdf6f274c4bbcd483ddf87f1787, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=36.0 K 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e4ebb46eb8247a68a598a48f789f8e8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a2abafdf6f274c4bbcd483ddf87f1787, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3683235bfd344384b851b36cb091ebe4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731993952823 2024-11-19T05:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f3bfe6bf86c94d45a27b34a742839fd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731993952823 2024-11-19T05:25:57,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9f3648671a42f58eb1e98341418831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:25:57,430 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 066f65af78134ae69f85ab8d7c2e23f5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:25:57,435 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:57,436 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:57,436 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/4c819c3e025b41f2b011f5d85847f1f3 is 50, key is test_row_0/B:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:57,437 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411199ed00f930bed4c7f9e7aadf7f2975843_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:57,438 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411199ed00f930bed4c7f9e7aadf7f2975843_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:57,438 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199ed00f930bed4c7f9e7aadf7f2975843_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742212_1388 (size=4469) 2024-11-19T05:25:57,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742213_1389 (size=12663) 2024-11-19T05:25:57,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-19T05:25:57,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:57,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:57,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-19T05:25:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T05:25:57,660 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:57,660 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:57,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T05:25:57,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-19T05:25:57,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:57,812 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:25:57,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111963c8b989bde642fb94ee5c957d152fc2_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993956198/Put/seqid=0 2024-11-19T05:25:57,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742214_1390 (size=12304) 2024-11-19T05:25:57,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:57,826 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111963c8b989bde642fb94ee5c957d152fc2_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111963c8b989bde642fb94ee5c957d152fc2_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/084e2eae2b944dbe85efc196678597c4, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:57,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/084e2eae2b944dbe85efc196678597c4 is 175, key is test_row_0/A:col10/1731993956198/Put/seqid=0 2024-11-19T05:25:57,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742215_1391 (size=31105) 2024-11-19T05:25:57,847 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#325 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:57,848 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/612f5a88bdb64b16a2f7dab96cffe928 is 175, key is test_row_0/A:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:57,850 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/4c819c3e025b41f2b011f5d85847f1f3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/4c819c3e025b41f2b011f5d85847f1f3 2024-11-19T05:25:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742216_1392 (size=31617) 2024-11-19T05:25:57,855 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into 4c819c3e025b41f2b011f5d85847f1f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:57,855 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:57,855 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=13, startTime=1731993957428; duration=0sec 2024-11-19T05:25:57,855 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:25:57,855 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:25:57,855 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:25:57,856 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:25:57,856 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:25:57,856 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:57,856 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/f1f8fbcc2e5b4521bd1a8ae1ab433010, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=36.0 K 2024-11-19T05:25:57,857 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f1f8fbcc2e5b4521bd1a8ae1ab433010, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731993951631 2024-11-19T05:25:57,857 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d4e131c7394513a72831abd4aaaffd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731993952823 2024-11-19T05:25:57,857 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 96cc5268cc53450cbeb983725dffd9af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:25:57,858 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/612f5a88bdb64b16a2f7dab96cffe928 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928 2024-11-19T05:25:57,861 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into 612f5a88bdb64b16a2f7dab96cffe928(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:57,862 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:57,862 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=13, startTime=1731993957428; duration=0sec 2024-11-19T05:25:57,862 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:57,862 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:25:57,863 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#327 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:25:57,864 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/63ccd2e2837f46f38b7584765c96ea14 is 50, key is test_row_0/C:col10/1731993955017/Put/seqid=0 2024-11-19T05:25:57,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742217_1393 (size=12663) 2024-11-19T05:25:57,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T05:25:58,231 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/084e2eae2b944dbe85efc196678597c4 2024-11-19T05:25:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/56e94a5d3ae24066b05456d5fab72736 is 50, key is test_row_0/B:col10/1731993956198/Put/seqid=0 2024-11-19T05:25:58,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742218_1394 (size=12151) 2024-11-19T05:25:58,242 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/56e94a5d3ae24066b05456d5fab72736 2024-11-19T05:25:58,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/30418d0705a64195a41d53276ed07199 is 50, key is test_row_0/C:col10/1731993956198/Put/seqid=0 2024-11-19T05:25:58,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742219_1395 (size=12151) 2024-11-19T05:25:58,252 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/30418d0705a64195a41d53276ed07199 2024-11-19T05:25:58,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/084e2eae2b944dbe85efc196678597c4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4 2024-11-19T05:25:58,258 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4, entries=150, sequenceid=235, filesize=30.4 K 2024-11-19T05:25:58,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/56e94a5d3ae24066b05456d5fab72736 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736 2024-11-19T05:25:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,262 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T05:25:58,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/30418d0705a64195a41d53276ed07199 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,265 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199, entries=150, sequenceid=235, filesize=11.9 K 2024-11-19T05:25:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,266 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 54ee91950f7dd8cedd166132c7496a58 in 454ms, sequenceid=235, compaction requested=false 2024-11-19T05:25:58,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:58,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:58,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-19T05:25:58,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 608 msec 2024-11-19T05:25:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/63ccd2e2837f46f38b7584765c96ea14 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/63ccd2e2837f46f38b7584765c96ea14 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 612 msec 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into 63ccd2e2837f46f38b7584765c96ea14(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:58,275 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=13, startTime=1731993957430; duration=0sec 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:25:58,275 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:25:58,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:25:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-19T05:25:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,443 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T05:25:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dace1abc94824b9686c26ad315c39a58_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993958412/Put/seqid=0 2024-11-19T05:25:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742220_1396 (size=9814) 2024-11-19T05:25:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119dace1abc94824b9686c26ad315c39a58_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dace1abc94824b9686c26ad315c39a58_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7fa17325190f4d4a9fe279585c0c2d7b, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:25:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7fa17325190f4d4a9fe279585c0c2d7b is 175, key is test_row_0/A:col10/1731993958412/Put/seqid=0 2024-11-19T05:25:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742221_1397 (size=22459) 2024-11-19T05:25:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:58,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:25:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:25:58,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994018550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994018552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994018554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994018555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994018556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:25:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994018668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994018668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994018668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994018668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994018668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,879 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7fa17325190f4d4a9fe279585c0c2d7b 2024-11-19T05:25:58,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/244af0e585974b6da5be914397551193 is 50, key is test_row_0/B:col10/1731993958412/Put/seqid=0 2024-11-19T05:25:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994018877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994018877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994018878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994018878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742222_1398 (size=7365) 2024-11-19T05:25:58,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994018887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:58,896 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:25:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994019197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994019197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994019198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994019198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994019207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,289 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/244af0e585974b6da5be914397551193 2024-11-19T05:25:59,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ae39e15c25ee41428186aa1c1a1f0a88 is 50, key is test_row_0/C:col10/1731993958412/Put/seqid=0 2024-11-19T05:25:59,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742223_1399 (size=7365) 2024-11-19T05:25:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:25:59,699 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ae39e15c25ee41428186aa1c1a1f0a88 2024-11-19T05:25:59,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/7fa17325190f4d4a9fe279585c0c2d7b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b 2024-11-19T05:25:59,705 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b, entries=100, sequenceid=243, filesize=21.9 K 2024-11-19T05:25:59,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/244af0e585974b6da5be914397551193 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193 2024-11-19T05:25:59,709 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193, entries=50, sequenceid=243, filesize=7.2 K 2024-11-19T05:25:59,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ae39e15c25ee41428186aa1c1a1f0a88 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88 2024-11-19T05:25:59,712 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88, entries=50, sequenceid=243, filesize=7.2 K 2024-11-19T05:25:59,713 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=194.56 KB/199230 for 54ee91950f7dd8cedd166132c7496a58 in 1271ms, sequenceid=243, compaction requested=true 2024-11-19T05:25:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:25:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:25:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-19T05:25:59,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-19T05:25:59,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:25:59,715 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:25:59,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-19T05:25:59,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-19T05:25:59,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4240 sec 2024-11-19T05:25:59,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994019707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994019707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.4290 sec 2024-11-19T05:25:59,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994019716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b1d9de25b874b02a2f9ee55ed2b7498_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_0/A:col10/1731993959714/Put/seqid=0 2024-11-19T05:25:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994019717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742224_1400 (size=14994) 2024-11-19T05:25:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994019727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:25:59,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:25:59,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994019827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:25:59,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:25:59,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994020037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,126 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:00,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199b1d9de25b874b02a2f9ee55ed2b7498_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b1d9de25b874b02a2f9ee55ed2b7498_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:00,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/db2e932047854434ac9616dbf7c551a1, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:00,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/db2e932047854434ac9616dbf7c551a1 is 175, key is test_row_0/A:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:00,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742225_1401 (size=39949) 2024-11-19T05:26:00,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994020347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-19T05:26:00,437 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:00,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-19T05:26:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T05:26:00,439 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:00,440 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:00,440 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:00,533 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=67.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/db2e932047854434ac9616dbf7c551a1 2024-11-19T05:26:00,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/2f16af315e184707b6be9b59dc07211a is 50, key is test_row_0/B:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:00,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742226_1402 (size=12301) 2024-11-19T05:26:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T05:26:00,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41884 deadline: 1731994020727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41840 deadline: 1731994020727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,732 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 54ee91950f7dd8cedd166132c7496a58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,733 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 54ee91950f7dd8cedd166132c7496a58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41858 deadline: 1731994020737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41806 deadline: 1731994020738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,739 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 54ee91950f7dd8cedd166132c7496a58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,740 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 54ee91950f7dd8cedd166132c7496a58 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:00,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:00,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T05:26:00,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:00,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41866 deadline: 1731994020857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:00,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:00,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:00,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:00,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:00,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:00,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/2f16af315e184707b6be9b59dc07211a 2024-11-19T05:26:00,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a826f421f7ee4ec58c834c290dcff270 is 50, key is test_row_0/C:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742227_1403 (size=12301) 2024-11-19T05:26:01,031 INFO [Thread-1516 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1516. 2024-11-19T05:26:01,031 DEBUG [Thread-1516 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,031 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,031 DEBUG [Thread-1516 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,031 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:01,032 INFO [Thread-1512 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1512. 2024-11-19T05:26:01,032 DEBUG [Thread-1512 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,032 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,033 DEBUG [Thread-1512 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:01,033 INFO [Thread-1510 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1510. 2024-11-19T05:26:01,033 DEBUG [Thread-1510 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,033 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,033 DEBUG [Thread-1510 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:01,038 INFO [Thread-1508 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1508. 2024-11-19T05:26:01,038 DEBUG [Thread-1508 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,038 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,038 DEBUG [Thread-1508 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:01,039 INFO [Thread-1514 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1514. 2024-11-19T05:26:01,039 DEBUG [Thread-1514 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,039 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,039 DEBUG [Thread-1514 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:01,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T05:26:01,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:01,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a826f421f7ee4ec58c834c290dcff270 2024-11-19T05:26:01,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/db2e932047854434ac9616dbf7c551a1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1 2024-11-19T05:26:01,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. as already flushing 2024-11-19T05:26:01,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:01,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1, entries=200, sequenceid=276, filesize=39.0 K 2024-11-19T05:26:01,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/2f16af315e184707b6be9b59dc07211a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a 2024-11-19T05:26:01,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T05:26:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/a826f421f7ee4ec58c834c290dcff270 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270 2024-11-19T05:26:01,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270, entries=150, sequenceid=276, filesize=12.0 K 2024-11-19T05:26:01,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for 54ee91950f7dd8cedd166132c7496a58 in 1656ms, sequenceid=276, compaction requested=true 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:01,371 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 54ee91950f7dd8cedd166132c7496a58:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:01,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:01,371 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/A is initiating minor compaction (all files) 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/B is initiating minor compaction (all files) 2024-11-19T05:26:01,372 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/A in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,372 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/B in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,372 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/4c819c3e025b41f2b011f5d85847f1f3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=43.4 K 2024-11-19T05:26:01,372 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=122.2 K 2024-11-19T05:26:01,372 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1] 2024-11-19T05:26:01,372 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c819c3e025b41f2b011f5d85847f1f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 612f5a88bdb64b16a2f7dab96cffe928, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56e94a5d3ae24066b05456d5fab72736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993956197 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 084e2eae2b944dbe85efc196678597c4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993956197 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 244af0e585974b6da5be914397551193, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731993958412 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa17325190f4d4a9fe279585c0c2d7b, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731993958397 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f16af315e184707b6be9b59dc07211a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731993958540 2024-11-19T05:26:01,373 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting db2e932047854434ac9616dbf7c551a1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731993958540 2024-11-19T05:26:01,378 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:01,379 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#B#compaction#336 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-19T05:26:01,380 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/eb49f411d82a49f3af374e005c7ccc37 is 50, key is test_row_0/B:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:01,380 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119fc04b6e0e4484d00bf92fe64ceec182c_54ee91950f7dd8cedd166132c7496a58 store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:01,383 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119fc04b6e0e4484d00bf92fe64ceec182c_54ee91950f7dd8cedd166132c7496a58, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:01,384 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119fc04b6e0e4484d00bf92fe64ceec182c_54ee91950f7dd8cedd166132c7496a58 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:01,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742228_1404 (size=12949) 2024-11-19T05:26:01,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742229_1405 (size=4469) 2024-11-19T05:26:01,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-19T05:26:01,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:26:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-19T05:26:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-19T05:26:01,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-19T05:26:01,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-19T05:26:01,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0830 sec 2024-11-19T05:26:01,553 DEBUG [regionserver/f990f635ffd0:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-19T05:26:01,555 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5] to archive 2024-11-19T05:26:01,556 WARN [regionserver/f990f635ffd0:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 2, running: 1 2024-11-19T05:26:01,557 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:01,559 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/44c113e40aef4f81a9e5b3a52a416bd1 2024-11-19T05:26:01,561 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/26d3d13225ed4d07be39f59484b18aa7 2024-11-19T05:26:01,562 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/109081d8ff5b48638209e915e41d4fbe 2024-11-19T05:26:01,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/d91bfbcc9ea548babe2c08dab58a7565 2024-11-19T05:26:01,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/fa99ea23b4da47b689969f1477c570a8 2024-11-19T05:26:01,565 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/c69cb65328264033b844f09205995cae 2024-11-19T05:26:01,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7297594f868c4549aa6f713f53abf85c 2024-11-19T05:26:01,567 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/f2e012429ef84b84af664645690b68be 2024-11-19T05:26:01,568 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/e4d2bfec313f43dd996878f28c795256 2024-11-19T05:26:01,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/2e308d5923d74fcf994ed2c60c726851 2024-11-19T05:26:01,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/ec5c3b23d3814edcb85580fa9edbda62 2024-11-19T05:26:01,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/8520fbbd80744769802d017f4b05c25c 2024-11-19T05:26:01,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3e4ebb46eb8247a68a598a48f789f8e8 2024-11-19T05:26:01,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3683235bfd344384b851b36cb091ebe4 2024-11-19T05:26:01,573 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/066f65af78134ae69f85ab8d7c2e23f5 2024-11-19T05:26:01,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/88061b3c92754f2fa9a7d6dde52738e7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/ffdab3e48db7448b8f12b1e1e2ee4eca, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/27fefea31fa34d05b3087208cad50567, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/a2abafdf6f274c4bbcd483ddf87f1787, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831] to archive 2024-11-19T05:26:01,575 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:01,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/1c6eb646fb2e4e7f9a1c7c5281e69594 2024-11-19T05:26:01,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/d491e30e12984db9a5733306568d63a1 2024-11-19T05:26:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-19T05:26:01,577 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:01,578 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/88061b3c92754f2fa9a7d6dde52738e7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/88061b3c92754f2fa9a7d6dde52738e7 2024-11-19T05:26:01,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/fee2082fbb6145238c93a27295e9afc1 2024-11-19T05:26:01,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/6032bf113cf04bb1ae576770d9306e98 2024-11-19T05:26:01,580 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/ffdab3e48db7448b8f12b1e1e2ee4eca to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/ffdab3e48db7448b8f12b1e1e2ee4eca 2024-11-19T05:26:01,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c4b4fed21f374bdb99a819d7280bc9a0 2024-11-19T05:26:01,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b2ea7912af7d4001b2f4b24edd054cee 2024-11-19T05:26:01,582 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/27fefea31fa34d05b3087208cad50567 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/27fefea31fa34d05b3087208cad50567 2024-11-19T05:26:01,583 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/b74d3d517203412bb99ee80d2873166e 2024-11-19T05:26:01,584 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/e26255af360e4aa480e9be06ad4088f9 2024-11-19T05:26:01,585 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/a2abafdf6f274c4bbcd483ddf87f1787 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/a2abafdf6f274c4bbcd483ddf87f1787 2024-11-19T05:26:01,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/c2152dccd6b34688b357cb2436b8342b 2024-11-19T05:26:01,586 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/f3bfe6bf86c94d45a27b34a742839fd1 2024-11-19T05:26:01,587 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/9e9f3648671a42f58eb1e98341418831 2024-11-19T05:26:01,589 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/234473eb417f4406864938a54558e61a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/826e0bd0812143578ff2f844d30b43b6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/0b1d297478674ec59bb2ee7782a6f8dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/f1f8fbcc2e5b4521bd1a8ae1ab433010, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af] to archive 2024-11-19T05:26:01,590 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:01,591 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/2921fc95b45d443da378fcb514cdd138 2024-11-19T05:26:01,593 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/d7ff021e06084a72b4a722caa0db44cb 2024-11-19T05:26:01,594 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/234473eb417f4406864938a54558e61a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/234473eb417f4406864938a54558e61a 2024-11-19T05:26:01,595 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/27e6c1f821ee46c1aca0f6c6f6d9dac8 2024-11-19T05:26:01,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/67412190c70a4a6cb2a67529d0e87e67 2024-11-19T05:26:01,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/826e0bd0812143578ff2f844d30b43b6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/826e0bd0812143578ff2f844d30b43b6 2024-11-19T05:26:01,598 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/14830721f6e3447a93055f2bd19f65a3 2024-11-19T05:26:01,599 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3a46f248f94f4c9e930ab4cbd05cf8cb 2024-11-19T05:26:01,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/0b1d297478674ec59bb2ee7782a6f8dd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/0b1d297478674ec59bb2ee7782a6f8dd 2024-11-19T05:26:01,600 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/72526761d5e54a429005e9f98acc2bf4 2024-11-19T05:26:01,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/cf2d24781e1840f7bd5fba9a6f861371 2024-11-19T05:26:01,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/f1f8fbcc2e5b4521bd1a8ae1ab433010 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/f1f8fbcc2e5b4521bd1a8ae1ab433010 2024-11-19T05:26:01,602 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/c7d5b3615e0a498e84050d7b6555f4ee 2024-11-19T05:26:01,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a5d4e131c7394513a72831abd4aaaffd 2024-11-19T05:26:01,603 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/96cc5268cc53450cbeb983725dffd9af 2024-11-19T05:26:01,789 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#A#compaction#337 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:01,791 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/85b5a0c765f7413b84daedd76005c010 is 175, key is test_row_0/A:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:01,797 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/eb49f411d82a49f3af374e005c7ccc37 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/eb49f411d82a49f3af374e005c7ccc37 2024-11-19T05:26:01,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742230_1406 (size=31393) 2024-11-19T05:26:01,802 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/B of 54ee91950f7dd8cedd166132c7496a58 into eb49f411d82a49f3af374e005c7ccc37(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:01,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:26:01,802 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/B, priority=12, startTime=1731993961371; duration=0sec 2024-11-19T05:26:01,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:01,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:B 2024-11-19T05:26:01,802 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:01,804 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 44480 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:01,804 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 54ee91950f7dd8cedd166132c7496a58/C is initiating minor compaction (all files) 2024-11-19T05:26:01,804 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 54ee91950f7dd8cedd166132c7496a58/C in TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:01,804 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/63ccd2e2837f46f38b7584765c96ea14, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp, totalSize=43.4 K 2024-11-19T05:26:01,805 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ccd2e2837f46f38b7584765c96ea14, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731993955015 2024-11-19T05:26:01,805 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30418d0705a64195a41d53276ed07199, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731993956197 2024-11-19T05:26:01,805 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae39e15c25ee41428186aa1c1a1f0a88, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731993958412 2024-11-19T05:26:01,806 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a826f421f7ee4ec58c834c290dcff270, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731993958540 2024-11-19T05:26:01,815 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 54ee91950f7dd8cedd166132c7496a58#C#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:01,816 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/3ec20136b8294fb9a7bb853306a72ff1 is 50, key is test_row_0/C:col10/1731993959714/Put/seqid=0 2024-11-19T05:26:01,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742231_1407 (size=12439) 2024-11-19T05:26:01,870 INFO [Thread-1499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1499. 2024-11-19T05:26:01,870 DEBUG [Thread-1499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:01,870 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,871 DEBUG [Thread-1499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:01,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,203 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/85b5a0c765f7413b84daedd76005c010 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/85b5a0c765f7413b84daedd76005c010 2024-11-19T05:26:02,208 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/A of 54ee91950f7dd8cedd166132c7496a58 into 85b5a0c765f7413b84daedd76005c010(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:02,208 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:26:02,208 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/A, priority=12, startTime=1731993961371; duration=0sec 2024-11-19T05:26:02,208 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:02,208 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:A 2024-11-19T05:26:02,224 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/3ec20136b8294fb9a7bb853306a72ff1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3ec20136b8294fb9a7bb853306a72ff1 2024-11-19T05:26:02,227 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 54ee91950f7dd8cedd166132c7496a58/C of 54ee91950f7dd8cedd166132c7496a58 into 3ec20136b8294fb9a7bb853306a72ff1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:02,227 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 54ee91950f7dd8cedd166132c7496a58: 2024-11-19T05:26:02,227 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., storeName=54ee91950f7dd8cedd166132c7496a58/C, priority=12, startTime=1731993961371; duration=0sec 2024-11-19T05:26:02,227 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:02,227 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 54ee91950f7dd8cedd166132c7496a58:C 2024-11-19T05:26:02,741 INFO [Thread-1503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1503. 2024-11-19T05:26:02,741 DEBUG [Thread-1503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:02,741 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,742 DEBUG [Thread-1503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,750 INFO [Thread-1501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1501. 2024-11-19T05:26:02,750 DEBUG [Thread-1501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:02,750 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,750 INFO [Thread-1497 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1497. 2024-11-19T05:26:02,750 DEBUG [Thread-1497 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:02,750 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,750 DEBUG [Thread-1501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,750 DEBUG [Thread-1497 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,760 INFO [Thread-1505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1505. 2024-11-19T05:26:02,760 DEBUG [Thread-1505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:02,760 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,760 DEBUG [Thread-1505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,760 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1246 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3735 rows 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1238 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3714 rows 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1234 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3699 rows 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1250 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3746 rows 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1236 2024-11-19T05:26:02,761 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3708 rows 2024-11-19T05:26:02,761 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:26:02,761 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:02,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:02,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:02,763 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:26:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:02,773 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993962772"}]},"ts":"1731993962772"} 2024-11-19T05:26:02,774 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:26:02,774 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:26:02,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:26:02,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, UNASSIGN}] 2024-11-19T05:26:02,776 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, UNASSIGN 2024-11-19T05:26:02,777 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:02,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, UNASSIGN because future has completed 2024-11-19T05:26:02,778 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:26:02,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:02,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:02,931 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:02,931 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:26:02,931 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 54ee91950f7dd8cedd166132c7496a58, disabling compactions & flushes 2024-11-19T05:26:02,931 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:02,931 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:02,931 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. after waiting 0 ms 2024-11-19T05:26:02,931 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:02,931 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 54ee91950f7dd8cedd166132c7496a58 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-19T05:26:02,943 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e59bae49e6d45b5a7eed7b8e50ff7ee_54ee91950f7dd8cedd166132c7496a58 is 50, key is test_row_1/A:col10/1731993962758/Put/seqid=0 2024-11-19T05:26:02,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742232_1408 (size=9914) 2024-11-19T05:26:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:03,348 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:03,358 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411199e59bae49e6d45b5a7eed7b8e50ff7ee_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e59bae49e6d45b5a7eed7b8e50ff7ee_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:03,360 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3eb3fb42808e49deb43dd572e889b034, store: [table=TestAcidGuarantees family=A region=54ee91950f7dd8cedd166132c7496a58] 2024-11-19T05:26:03,361 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3eb3fb42808e49deb43dd572e889b034 is 175, key is test_row_1/A:col10/1731993962758/Put/seqid=0 2024-11-19T05:26:03,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742233_1409 (size=22561) 2024-11-19T05:26:03,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:03,767 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3eb3fb42808e49deb43dd572e889b034 2024-11-19T05:26:03,779 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/bd14cc7a35524feea5c9daadfbdd92e6 is 50, key is test_row_1/B:col10/1731993962758/Put/seqid=0 2024-11-19T05:26:03,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742234_1410 (size=9857) 2024-11-19T05:26:03,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:04,185 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/bd14cc7a35524feea5c9daadfbdd92e6 2024-11-19T05:26:04,198 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ec11035778b14813ade60de847d8e4e3 is 50, key is test_row_1/C:col10/1731993962758/Put/seqid=0 2024-11-19T05:26:04,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742235_1411 (size=9857) 2024-11-19T05:26:04,604 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ec11035778b14813ade60de847d8e4e3 2024-11-19T05:26:04,612 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/A/3eb3fb42808e49deb43dd572e889b034 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3eb3fb42808e49deb43dd572e889b034 2024-11-19T05:26:04,620 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3eb3fb42808e49deb43dd572e889b034, entries=100, sequenceid=288, filesize=22.0 K 2024-11-19T05:26:04,622 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/B/bd14cc7a35524feea5c9daadfbdd92e6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/bd14cc7a35524feea5c9daadfbdd92e6 2024-11-19T05:26:04,627 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/bd14cc7a35524feea5c9daadfbdd92e6, entries=100, sequenceid=288, filesize=9.6 K 2024-11-19T05:26:04,628 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/.tmp/C/ec11035778b14813ade60de847d8e4e3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ec11035778b14813ade60de847d8e4e3 2024-11-19T05:26:04,632 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ec11035778b14813ade60de847d8e4e3, entries=100, sequenceid=288, filesize=9.6 K 2024-11-19T05:26:04,633 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 54ee91950f7dd8cedd166132c7496a58 in 1702ms, sequenceid=288, compaction requested=false 2024-11-19T05:26:04,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1] to archive 2024-11-19T05:26:04,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:04,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/612f5a88bdb64b16a2f7dab96cffe928 2024-11-19T05:26:04,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/084e2eae2b944dbe85efc196678597c4 2024-11-19T05:26:04,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/7fa17325190f4d4a9fe279585c0c2d7b 2024-11-19T05:26:04,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/db2e932047854434ac9616dbf7c551a1 2024-11-19T05:26:04,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/4c819c3e025b41f2b011f5d85847f1f3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a] to archive 2024-11-19T05:26:04,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:04,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/4c819c3e025b41f2b011f5d85847f1f3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/4c819c3e025b41f2b011f5d85847f1f3 2024-11-19T05:26:04,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/56e94a5d3ae24066b05456d5fab72736 2024-11-19T05:26:04,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/244af0e585974b6da5be914397551193 2024-11-19T05:26:04,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/2f16af315e184707b6be9b59dc07211a 2024-11-19T05:26:04,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/63ccd2e2837f46f38b7584765c96ea14, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270] to archive 2024-11-19T05:26:04,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:04,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/63ccd2e2837f46f38b7584765c96ea14 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/63ccd2e2837f46f38b7584765c96ea14 2024-11-19T05:26:04,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/30418d0705a64195a41d53276ed07199 2024-11-19T05:26:04,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ae39e15c25ee41428186aa1c1a1f0a88 2024-11-19T05:26:04,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/a826f421f7ee4ec58c834c290dcff270 2024-11-19T05:26:04,650 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits/291.seqid, newMaxSeqId=291, maxSeqId=4 2024-11-19T05:26:04,650 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. 2024-11-19T05:26:04,650 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 54ee91950f7dd8cedd166132c7496a58: Waiting for close lock at 1731993962931Running coprocessor pre-close hooks at 1731993962931Disabling compacts and flushes for region at 1731993962931Disabling writes for close at 1731993962931Obtaining lock to block concurrent updates at 1731993962931Preparing flush snapshotting stores in 54ee91950f7dd8cedd166132c7496a58 at 1731993962931Finished memstore snapshotting TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731993962932 (+1 ms)Flushing stores of TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58. at 1731993962933 (+1 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/A: creating writer at 1731993962933Flushing mob file 54ee91950f7dd8cedd166132c7496a58/A: appending metadata at 1731993962943 (+10 ms)Flushing mob file 54ee91950f7dd8cedd166132c7496a58/A: closing flushed file at 1731993962943Flushing 54ee91950f7dd8cedd166132c7496a58/A: appending metadata at 1731993963360 (+417 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/A: closing flushed file at 1731993963360Flushing 54ee91950f7dd8cedd166132c7496a58/B: creating writer at 1731993963776 (+416 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/B: appending metadata at 1731993963779 (+3 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/B: closing flushed file at 1731993963779Flushing 54ee91950f7dd8cedd166132c7496a58/C: creating writer at 1731993964195 (+416 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/C: appending metadata at 1731993964198 (+3 ms)Flushing 54ee91950f7dd8cedd166132c7496a58/C: closing flushed file at 1731993964198Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@204810ff: reopening flushed file at 1731993964611 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@391ffa5f: reopening flushed file at 1731993964621 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d05841: reopening flushed file at 1731993964627 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 54ee91950f7dd8cedd166132c7496a58 in 1702ms, sequenceid=288, compaction requested=false at 1731993964633 (+6 ms)Writing region close event to WAL at 1731993964648 (+15 ms)Running coprocessor post-close hooks at 1731993964650 (+2 ms)Closed at 1731993964650 2024-11-19T05:26:04,652 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,652 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=54ee91950f7dd8cedd166132c7496a58, regionState=CLOSED 2024-11-19T05:26:04,653 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:04,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-19T05:26:04,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 54ee91950f7dd8cedd166132c7496a58, server=f990f635ffd0,38449,1731993838792 in 1.8750 sec 2024-11-19T05:26:04,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-19T05:26:04,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=54ee91950f7dd8cedd166132c7496a58, UNASSIGN in 1.8790 sec 2024-11-19T05:26:04,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-19T05:26:04,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8820 sec 2024-11-19T05:26:04,658 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993964657"}]},"ts":"1731993964657"} 2024-11-19T05:26:04,658 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:26:04,659 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:26:04,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-11-19T05:26:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-19T05:26:04,919 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:04,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:26:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:04,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:04,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T05:26:04,927 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:04,931 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,935 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits] 2024-11-19T05:26:04,939 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3eb3fb42808e49deb43dd572e889b034 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/3eb3fb42808e49deb43dd572e889b034 2024-11-19T05:26:04,940 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/85b5a0c765f7413b84daedd76005c010 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/A/85b5a0c765f7413b84daedd76005c010 2024-11-19T05:26:04,943 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/bd14cc7a35524feea5c9daadfbdd92e6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/bd14cc7a35524feea5c9daadfbdd92e6 2024-11-19T05:26:04,945 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/eb49f411d82a49f3af374e005c7ccc37 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/B/eb49f411d82a49f3af374e005c7ccc37 2024-11-19T05:26:04,948 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3ec20136b8294fb9a7bb853306a72ff1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/3ec20136b8294fb9a7bb853306a72ff1 2024-11-19T05:26:04,950 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ec11035778b14813ade60de847d8e4e3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/C/ec11035778b14813ade60de847d8e4e3 2024-11-19T05:26:04,953 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits/291.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58/recovered.edits/291.seqid 2024-11-19T05:26:04,954 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,954 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:26:04,955 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:26:04,956 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T05:26:04,960 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119004bde8d274e4c16bbd4ad4bc8a7dfd8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119004bde8d274e4c16bbd4ad4bc8a7dfd8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,962 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193049ff6fa6f14da2a4d221e049702567_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193049ff6fa6f14da2a4d221e049702567_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,964 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119330ad4a0bd0f44adaf9cc61fdda0117f_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119330ad4a0bd0f44adaf9cc61fdda0117f_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,966 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f9f428effc84593a7da425be1b04e03_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194f9f428effc84593a7da425be1b04e03_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,968 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cf7bbaaf5d84eb9ab7a843ee195deb7_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195cf7bbaaf5d84eb9ab7a843ee195deb7_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,970 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111963c8b989bde642fb94ee5c957d152fc2_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111963c8b989bde642fb94ee5c957d152fc2_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,972 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974c0f8c9c94e4851999c8070d9c598e0_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111974c0f8c9c94e4851999c8070d9c598e0_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,974 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e4d970c3a6942c4b86986d1d3cc10f8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198e4d970c3a6942c4b86986d1d3cc10f8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,976 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b1d9de25b874b02a2f9ee55ed2b7498_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199b1d9de25b874b02a2f9ee55ed2b7498_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,978 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e59bae49e6d45b5a7eed7b8e50ff7ee_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411199e59bae49e6d45b5a7eed7b8e50ff7ee_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,980 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9770c1b4c894239bc2511156404cc08_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9770c1b4c894239bc2511156404cc08_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,982 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4be75ee893b418bb50233834be98f50_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c4be75ee893b418bb50233834be98f50_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,984 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c8635f4c0695431b9b798f283bac272a_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119c8635f4c0695431b9b798f283bac272a_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,986 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dace1abc94824b9686c26ad315c39a58_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119dace1abc94824b9686c26ad315c39a58_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,988 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f57ee0b022124dd59404c2bbc1752de8_54ee91950f7dd8cedd166132c7496a58 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119f57ee0b022124dd59404c2bbc1752de8_54ee91950f7dd8cedd166132c7496a58 2024-11-19T05:26:04,988 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:26:04,991 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:04,993 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:26:04,995 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:26:04,997 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:04,997 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:26:04,997 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731993964997"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:04,999 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:26:04,999 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 54ee91950f7dd8cedd166132c7496a58, NAME => 'TestAcidGuarantees,,1731993938862.54ee91950f7dd8cedd166132c7496a58.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:26:04,999 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:26:04,999 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731993964999"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:05,001 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:26:05,002 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:05,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-19T05:26:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-19T05:26:05,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:26:05,040 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:05,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=242 (was 237) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1179197976_22 at /127.0.0.1:43094 [Waiting for operation #554] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-299407974_22 at /127.0.0.1:41762 [Waiting for operation #472] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-299407974_22 at /127.0.0.1:47306 [Waiting for operation #352] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1179197976_22 at /127.0.0.1:47300 [Waiting for operation #360] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/f990f635ffd0:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=534 (was 449) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11806 (was 12795) 2024-11-19T05:26:05,059 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=242, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=534, ProcessCount=11, AvailableMemoryMB=11805 2024-11-19T05:26:05,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:26:05,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:26:05,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:05,063 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:26:05,063 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:05,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-19T05:26:05,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:05,064 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:26:05,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742236_1412 (size=959) 2024-11-19T05:26:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:05,476 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:26:05,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742237_1413 (size=53) 2024-11-19T05:26:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:05,888 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:26:05,888 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c8584c7fbd7ef0113dc3083d3d909de9, disabling compactions & flushes 2024-11-19T05:26:05,888 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:05,888 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:05,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. after waiting 0 ms 2024-11-19T05:26:05,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:05,889 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:05,889 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8584c7fbd7ef0113dc3083d3d909de9: Waiting for close lock at 1731993965888Disabling compacts and flushes for region at 1731993965888Disabling writes for close at 1731993965889 (+1 ms)Writing region close event to WAL at 1731993965889Closed at 1731993965889 2024-11-19T05:26:05,891 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:26:05,892 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993965891"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993965891"}]},"ts":"1731993965891"} 2024-11-19T05:26:05,895 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:26:05,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:26:05,897 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993965897"}]},"ts":"1731993965897"} 2024-11-19T05:26:05,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:26:05,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, ASSIGN}] 2024-11-19T05:26:05,901 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, ASSIGN 2024-11-19T05:26:05,902 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:26:06,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c8584c7fbd7ef0113dc3083d3d909de9, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:06,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, ASSIGN because future has completed 2024-11-19T05:26:06,056 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:06,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:06,217 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:06,218 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:26:06,219 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,219 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:26:06,220 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,220 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,222 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,223 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8584c7fbd7ef0113dc3083d3d909de9 columnFamilyName A 2024-11-19T05:26:06,223 DEBUG [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:06,224 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(327): Store=c8584c7fbd7ef0113dc3083d3d909de9/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:06,224 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,225 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8584c7fbd7ef0113dc3083d3d909de9 columnFamilyName B 2024-11-19T05:26:06,225 DEBUG [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:06,226 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(327): Store=c8584c7fbd7ef0113dc3083d3d909de9/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:06,226 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,227 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8584c7fbd7ef0113dc3083d3d909de9 columnFamilyName C 2024-11-19T05:26:06,227 DEBUG [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:06,228 INFO [StoreOpener-c8584c7fbd7ef0113dc3083d3d909de9-1 {}] regionserver.HStore(327): Store=c8584c7fbd7ef0113dc3083d3d909de9/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:06,228 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,229 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,229 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,230 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,230 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,231 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:26:06,232 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,234 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:26:06,235 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened c8584c7fbd7ef0113dc3083d3d909de9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70073528, jitterRate=0.04417693614959717}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:26:06,235 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:06,236 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for c8584c7fbd7ef0113dc3083d3d909de9: Running coprocessor pre-open hook at 1731993966220Writing region info on filesystem at 1731993966220Initializing all the Stores at 1731993966221 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993966221Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993966221Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993966221Cleaning up temporary data from old regions at 1731993966230 (+9 ms)Running coprocessor post-open hooks at 1731993966235 (+5 ms)Region opened successfully at 1731993966236 (+1 ms) 2024-11-19T05:26:06,236 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., pid=112, masterSystemTime=1731993966209 2024-11-19T05:26:06,238 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:06,238 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:06,239 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=c8584c7fbd7ef0113dc3083d3d909de9, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:06,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:06,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-19T05:26:06,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 in 185 msec 2024-11-19T05:26:06,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-19T05:26:06,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, ASSIGN in 344 msec 2024-11-19T05:26:06,245 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:26:06,245 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993966245"}]},"ts":"1731993966245"} 2024-11-19T05:26:06,247 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:26:06,248 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:26:06,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-19T05:26:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-19T05:26:07,219 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:07,222 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,222 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,224 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ff42b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,226 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba230e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:26:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:26:07,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26877170 2024-11-19T05:26:07,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:26:07,233 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:26:07,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,236 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56d45f34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3905d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,241 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6219697f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,242 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7017fce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,243 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,243 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,244 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dd9582b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,245 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,245 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477b6253, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,247 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,248 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5d0e3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,250 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cf76b8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db26627, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,254 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13659c91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380bb61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,257 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,257 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c31aeca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,258 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,259 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e00e5e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,259 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,260 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f7f2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,261 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@140aab0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,262 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,262 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46546030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,263 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b1b653, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb1bf9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:07,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:07,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1129cdcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:07,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:07,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6755ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:07,267 DEBUG [Thread-1883 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,267 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,267 DEBUG [Thread-1878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,267 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,267 DEBUG [Thread-1885 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,267 DEBUG [Thread-1889 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,268 DEBUG [Thread-1887 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,268 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,268 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,268 DEBUG [Thread-1881 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-19T05:26:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,269 DEBUG [Thread-1881 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,269 DEBUG [Thread-1881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:07,270 DEBUG [Thread-1876 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:07,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,270 DEBUG [Thread-1876 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:07,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,272 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:07,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:07,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:07,276 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2] 2024-11-19T05:26:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:07,282 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:07,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:07,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7b1985637603458892d72ec6f4b141a7 is 50, key is test_row_0/A:col10/1731993967280/Put/seqid=0 2024-11-19T05:26:07,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742238_1414 (size=9657) 2024-11-19T05:26:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994027312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994027318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994027319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994027320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994027321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:07,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:07,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994027427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994027427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994027437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994027437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994027437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994027637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994027637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994027647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994027647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994027647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7b1985637603458892d72ec6f4b141a7 2024-11-19T05:26:07,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/010292020cff4230825df24f85044af3 is 50, key is test_row_0/B:col10/1731993967280/Put/seqid=0 2024-11-19T05:26:07,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742239_1415 (size=9657) 2024-11-19T05:26:07,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:07,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:07,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994027947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994027947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994027957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994027957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994027957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:07,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,129 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/010292020cff4230825df24f85044af3 2024-11-19T05:26:08,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/ed25998e3e09495687fbbf180511d6a0 is 50, key is test_row_0/C:col10/1731993967280/Put/seqid=0 2024-11-19T05:26:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742240_1416 (size=9657) 2024-11-19T05:26:08,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:08,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:08,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:08,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994028457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994028467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994028467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994028467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994028477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:08,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:08,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:08,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:08,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:08,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/ed25998e3e09495687fbbf180511d6a0 2024-11-19T05:26:08,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7b1985637603458892d72ec6f4b141a7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7 2024-11-19T05:26:08,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7, entries=100, sequenceid=12, filesize=9.4 K 2024-11-19T05:26:08,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/010292020cff4230825df24f85044af3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3 2024-11-19T05:26:08,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3, entries=100, sequenceid=12, filesize=9.4 K 2024-11-19T05:26:08,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/ed25998e3e09495687fbbf180511d6a0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0 2024-11-19T05:26:08,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0, entries=100, sequenceid=12, filesize=9.4 K 2024-11-19T05:26:08,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c8584c7fbd7ef0113dc3083d3d909de9 in 1292ms, sequenceid=12, compaction requested=false 2024-11-19T05:26:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:08,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-19T05:26:08,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:08,648 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:08,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/accc060df67d4bf89b000893eef8e30b is 50, key is test_row_0/A:col10/1731993967319/Put/seqid=0 2024-11-19T05:26:08,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742241_1417 (size=12001) 2024-11-19T05:26:09,056 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/accc060df67d4bf89b000893eef8e30b 2024-11-19T05:26:09,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 is 50, key is test_row_0/B:col10/1731993967319/Put/seqid=0 2024-11-19T05:26:09,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742242_1418 (size=12001) 2024-11-19T05:26:09,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:09,466 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 2024-11-19T05:26:09,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/eda225a9781e438fa3bb4d81092d0ba2 is 50, key is test_row_0/C:col10/1731993967319/Put/seqid=0 2024-11-19T05:26:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:09,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:09,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742243_1419 (size=12001) 2024-11-19T05:26:09,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994029492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994029495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994029495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994029495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994029497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,503 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994029607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994029607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994029607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994029607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994029817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994029817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994029817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994029817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:09,878 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/eda225a9781e438fa3bb4d81092d0ba2 2024-11-19T05:26:09,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/accc060df67d4bf89b000893eef8e30b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b 2024-11-19T05:26:09,885 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:26:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 2024-11-19T05:26:09,889 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:26:09,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/eda225a9781e438fa3bb4d81092d0ba2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2 2024-11-19T05:26:09,892 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-19T05:26:09,893 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for c8584c7fbd7ef0113dc3083d3d909de9 in 1245ms, sequenceid=37, compaction requested=false 2024-11-19T05:26:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:09,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-19T05:26:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-19T05:26:09,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-19T05:26:09,896 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6230 sec 2024-11-19T05:26:09,897 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.6290 sec 2024-11-19T05:26:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:10,131 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:10,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:26:10,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/755f4723d6434b66ab3aa170966946c7 is 50, key is test_row_0/A:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742244_1420 (size=12001) 2024-11-19T05:26:10,167 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:26:10,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994030166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994030172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994030173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994030175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994030277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994030277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994030287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994030287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,295 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994030487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994030487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994030507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994030507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/755f4723d6434b66ab3aa170966946c7 2024-11-19T05:26:10,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/1e7e85e452ba42c396218ad6ffadd7bb is 50, key is test_row_0/B:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:10,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742245_1421 (size=12001) 2024-11-19T05:26:10,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994030797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994030807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994030817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994030817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:10,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:10,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:10,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/1e7e85e452ba42c396218ad6ffadd7bb 2024-11-19T05:26:10,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/acf76ec75b9d4dc18d01120a102c03fd is 50, key is test_row_0/C:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742246_1422 (size=12001) 2024-11-19T05:26:11,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994031317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994031327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994031328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994031328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/acf76ec75b9d4dc18d01120a102c03fd 2024-11-19T05:26:11,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/755f4723d6434b66ab3aa170966946c7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7 2024-11-19T05:26:11,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T05:26:11,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/1e7e85e452ba42c396218ad6ffadd7bb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb 2024-11-19T05:26:11,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T05:26:11,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/acf76ec75b9d4dc18d01120a102c03fd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd 2024-11-19T05:26:11,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd, entries=150, sequenceid=51, filesize=11.7 K 2024-11-19T05:26:11,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c8584c7fbd7ef0113dc3083d3d909de9 in 1248ms, sequenceid=51, compaction requested=true 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:11,379 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:11,379 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:11,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:11,380 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:11,380 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:11,380 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:11,380 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:11,380 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,380 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,380 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=32.9 K 2024-11-19T05:26:11,380 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=32.9 K 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b1985637603458892d72ec6f4b141a7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993967280 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 010292020cff4230825df24f85044af3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993967280 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ceeb7ad5bbd34da9a7c41a8b8fb3d7a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993967308 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting accc060df67d4bf89b000893eef8e30b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993967308 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 755f4723d6434b66ab3aa170966946c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:11,381 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e7e85e452ba42c396218ad6ffadd7bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:11,387 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:11,387 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:11,388 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/b7ccac96e7b5484aa37c2e539ab23d87 is 50, key is test_row_0/A:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:11,388 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/55933f718a55411a9f6b28592218a917 is 50, key is test_row_0/B:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:11,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742247_1423 (size=12104) 2024-11-19T05:26:11,401 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/55933f718a55411a9f6b28592218a917 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/55933f718a55411a9f6b28592218a917 2024-11-19T05:26:11,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742248_1424 (size=12104) 2024-11-19T05:26:11,427 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 55933f718a55411a9f6b28592218a917(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:11,427 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:11,427 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=13, startTime=1731993971379; duration=0sec 2024-11-19T05:26:11,427 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:11,427 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:11,427 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:11,428 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:11,428 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:11,428 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,428 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=32.9 K 2024-11-19T05:26:11,429 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ed25998e3e09495687fbbf180511d6a0, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731993967280 2024-11-19T05:26:11,429 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting eda225a9781e438fa3bb4d81092d0ba2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731993967308 2024-11-19T05:26:11,430 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting acf76ec75b9d4dc18d01120a102c03fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:11,432 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/b7ccac96e7b5484aa37c2e539ab23d87 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/b7ccac96e7b5484aa37c2e539ab23d87 2024-11-19T05:26:11,437 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into b7ccac96e7b5484aa37c2e539ab23d87(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:11,437 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:11,437 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=13, startTime=1731993971379; duration=0sec 2024-11-19T05:26:11,437 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:11,437 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-19T05:26:11,438 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:11,438 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:11,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-19T05:26:11,440 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:11,440 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:11,441 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/8bcaaec9f3774dc7a928f012209a999d is 50, key is test_row_0/C:col10/1731993970130/Put/seqid=0 2024-11-19T05:26:11,441 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:11,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:11,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742249_1425 (size=12104) 2024-11-19T05:26:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:11,518 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:11,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:26:11,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/168e569e027343ec9f78152336a1c4dd is 50, key is test_row_0/A:col10/1731993971517/Put/seqid=0 2024-11-19T05:26:11,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742250_1426 (size=14341) 2024-11-19T05:26:11,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/168e569e027343ec9f78152336a1c4dd 2024-11-19T05:26:11,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/3300cf9cd6564f8fb99f7194365277f9 is 50, key is test_row_0/B:col10/1731993971517/Put/seqid=0 2024-11-19T05:26:11,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742251_1427 (size=12001) 2024-11-19T05:26:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994031576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:11,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994031687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:11,861 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/8bcaaec9f3774dc7a928f012209a999d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/8bcaaec9f3774dc7a928f012209a999d 2024-11-19T05:26:11,865 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 8bcaaec9f3774dc7a928f012209a999d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:11,865 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:11,865 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=13, startTime=1731993971379; duration=0sec 2024-11-19T05:26:11,865 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:11,865 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:11,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:11,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:11,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:11,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994031897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:11,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/3300cf9cd6564f8fb99f7194365277f9 2024-11-19T05:26:11,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/72b24603143642a3bf787812669c97b7 is 50, key is test_row_0/C:col10/1731993971517/Put/seqid=0 2024-11-19T05:26:11,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742252_1428 (size=12001) 2024-11-19T05:26:12,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:12,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:12,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994032207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,212 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994032337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994032337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994032337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,342 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,342 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,343 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/72b24603143642a3bf787812669c97b7 2024-11-19T05:26:12,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/168e569e027343ec9f78152336a1c4dd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd 2024-11-19T05:26:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994032347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,353 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd, entries=200, sequenceid=76, filesize=14.0 K 2024-11-19T05:26:12,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/3300cf9cd6564f8fb99f7194365277f9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9 2024-11-19T05:26:12,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:12,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T05:26:12,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/72b24603143642a3bf787812669c97b7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7 2024-11-19T05:26:12,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-19T05:26:12,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c8584c7fbd7ef0113dc3083d3d909de9 in 849ms, sequenceid=76, compaction requested=false 2024-11-19T05:26:12,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:12,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-19T05:26:12,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,510 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:26:12,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/3478f0c079f241aeb9f8b03f5e582db2 is 50, key is test_row_0/A:col10/1731993971563/Put/seqid=0 2024-11-19T05:26:12,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742253_1429 (size=12001) 2024-11-19T05:26:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:12,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:12,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:12,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994032865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:12,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:12,936 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/3478f0c079f241aeb9f8b03f5e582db2 2024-11-19T05:26:12,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/07bcbbd9f4e940e2b64b5aba57301641 is 50, key is test_row_0/B:col10/1731993971563/Put/seqid=0 2024-11-19T05:26:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742254_1430 (size=12001) 2024-11-19T05:26:12,947 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/07bcbbd9f4e940e2b64b5aba57301641 2024-11-19T05:26:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5e0cf2a6e3b644bdbb090a56d80ba337 is 50, key is test_row_0/C:col10/1731993971563/Put/seqid=0 2024-11-19T05:26:12,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742255_1431 (size=12001) 2024-11-19T05:26:12,973 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5e0cf2a6e3b644bdbb090a56d80ba337 2024-11-19T05:26:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/3478f0c079f241aeb9f8b03f5e582db2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2 2024-11-19T05:26:12,983 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T05:26:12,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/07bcbbd9f4e940e2b64b5aba57301641 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641 2024-11-19T05:26:12,989 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T05:26:12,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5e0cf2a6e3b644bdbb090a56d80ba337 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337 2024-11-19T05:26:12,992 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337, entries=150, sequenceid=90, filesize=11.7 K 2024-11-19T05:26:12,993 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c8584c7fbd7ef0113dc3083d3d909de9 in 483ms, sequenceid=90, compaction requested=true 2024-11-19T05:26:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:12,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-19T05:26:12,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-19T05:26:12,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-19T05:26:12,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-11-19T05:26:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:12,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:12,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:26:13,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.5600 sec 2024-11-19T05:26:13,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/6c05582cff714b738c2475c09fe8736a is 50, key is test_row_0/A:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742256_1432 (size=14341) 2024-11-19T05:26:13,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/6c05582cff714b738c2475c09fe8736a 2024-11-19T05:26:13,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/df537d9d41934b81a077b071714cc4a3 is 50, key is test_row_0/B:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742257_1433 (size=12001) 2024-11-19T05:26:13,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/df537d9d41934b81a077b071714cc4a3 2024-11-19T05:26:13,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/1433881bfa6f4dad8d312e89f35867ae is 50, key is test_row_0/C:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742258_1434 (size=12001) 2024-11-19T05:26:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994033116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:13,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994033227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:13,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/1433881bfa6f4dad8d312e89f35867ae 2024-11-19T05:26:13,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/6c05582cff714b738c2475c09fe8736a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a 2024-11-19T05:26:13,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994033447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:13,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a, entries=200, sequenceid=114, filesize=14.0 K 2024-11-19T05:26:13,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/df537d9d41934b81a077b071714cc4a3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3 2024-11-19T05:26:13,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3, entries=150, sequenceid=114, filesize=11.7 K 2024-11-19T05:26:13,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/1433881bfa6f4dad8d312e89f35867ae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae 2024-11-19T05:26:13,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae, entries=150, sequenceid=114, filesize=11.7 K 2024-11-19T05:26:13,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c8584c7fbd7ef0113dc3083d3d909de9 in 469ms, sequenceid=114, compaction requested=true 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:13,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-19T05:26:13,468 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:13,468 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:13,471 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:13,471 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:13,471 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:13,471 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/8bcaaec9f3774dc7a928f012209a999d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=47.0 K 2024-11-19T05:26:13,471 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:13,471 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bcaaec9f3774dc7a928f012209a999d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:13,471 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:13,471 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:13,472 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/b7ccac96e7b5484aa37c2e539ab23d87, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=51.5 K 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b24603143642a3bf787812669c97b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731993970170 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7ccac96e7b5484aa37c2e539ab23d87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e0cf2a6e3b644bdbb090a56d80ba337, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731993971544 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 168e569e027343ec9f78152336a1c4dd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731993970170 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1433881bfa6f4dad8d312e89f35867ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972860 2024-11-19T05:26:13,472 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3478f0c079f241aeb9f8b03f5e582db2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731993971544 2024-11-19T05:26:13,473 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c05582cff714b738c2475c09fe8736a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972811 2024-11-19T05:26:13,479 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:13,480 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/9de59f6272d645cfafde246b2984fb5e is 50, key is test_row_0/A:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,482 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:13,482 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5f4c86f3607a4456b055456a2b055dc3 is 50, key is test_row_0/C:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742259_1435 (size=12241) 2024-11-19T05:26:13,497 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5f4c86f3607a4456b055456a2b055dc3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5f4c86f3607a4456b055456a2b055dc3 2024-11-19T05:26:13,502 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 5f4c86f3607a4456b055456a2b055dc3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:13,502 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:13,502 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=12, startTime=1731993973467; duration=0sec 2024-11-19T05:26:13,502 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:13,502 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:13,502 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:13,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742260_1436 (size=12241) 2024-11-19T05:26:13,507 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:13,507 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:13,507 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:13,507 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/55933f718a55411a9f6b28592218a917, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=47.0 K 2024-11-19T05:26:13,507 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 55933f718a55411a9f6b28592218a917, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731993969494 2024-11-19T05:26:13,508 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 3300cf9cd6564f8fb99f7194365277f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731993970170 2024-11-19T05:26:13,508 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 07bcbbd9f4e940e2b64b5aba57301641, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731993971544 2024-11-19T05:26:13,510 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting df537d9d41934b81a077b071714cc4a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972860 2024-11-19T05:26:13,512 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/9de59f6272d645cfafde246b2984fb5e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9de59f6272d645cfafde246b2984fb5e 2024-11-19T05:26:13,521 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into 9de59f6272d645cfafde246b2984fb5e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:13,521 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:13,521 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=12, startTime=1731993973467; duration=0sec 2024-11-19T05:26:13,521 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:13,521 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:13,524 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#365 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:13,525 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/96b92f467de4480ba3e075f03af53791 is 50, key is test_row_0/B:col10/1731993972860/Put/seqid=0 2024-11-19T05:26:13,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742261_1437 (size=12241) 2024-11-19T05:26:13,544 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/96b92f467de4480ba3e075f03af53791 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/96b92f467de4480ba3e075f03af53791 2024-11-19T05:26:13,550 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 96b92f467de4480ba3e075f03af53791(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:13,551 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:13,551 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=12, startTime=1731993973467; duration=0sec 2024-11-19T05:26:13,551 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:13,551 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-19T05:26:13,587 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:13,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-19T05:26:13,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T05:26:13,590 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:13,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T05:26:13,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-19T05:26:13,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:13,743 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:26:13,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c4db25aad2264c5f82ee2c4903d113e9 is 50, key is test_row_0/A:col10/1731993973083/Put/seqid=0 2024-11-19T05:26:13,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742262_1438 (size=12001) 2024-11-19T05:26:13,758 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c4db25aad2264c5f82ee2c4903d113e9 2024-11-19T05:26:13,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/d0ae54bdc65c499f92420d67e8550563 is 50, key is test_row_0/B:col10/1731993973083/Put/seqid=0 2024-11-19T05:26:13,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:13,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:13,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742263_1439 (size=12001) 2024-11-19T05:26:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T05:26:13,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994033945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:13,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994034068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,183 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/d0ae54bdc65c499f92420d67e8550563 2024-11-19T05:26:14,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/58f059dae50440bd9296af43c0d5e78a is 50, key is test_row_0/C:col10/1731993973083/Put/seqid=0 2024-11-19T05:26:14,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742264_1440 (size=12001) 2024-11-19T05:26:14,195 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/58f059dae50440bd9296af43c0d5e78a 2024-11-19T05:26:14,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c4db25aad2264c5f82ee2c4903d113e9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9 2024-11-19T05:26:14,204 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T05:26:14,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/d0ae54bdc65c499f92420d67e8550563 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563 2024-11-19T05:26:14,208 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T05:26:14,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/58f059dae50440bd9296af43c0d5e78a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a 2024-11-19T05:26:14,215 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a, entries=150, sequenceid=129, filesize=11.7 K 2024-11-19T05:26:14,216 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c8584c7fbd7ef0113dc3083d3d909de9 in 472ms, sequenceid=129, compaction requested=false 2024-11-19T05:26:14,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:14,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-19T05:26:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T05:26:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-19T05:26:14,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-19T05:26:14,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 628 msec 2024-11-19T05:26:14,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 633 msec 2024-11-19T05:26:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:14,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:14,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:14,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/84322771f600497dbf00367ba328fe37 is 50, key is test_row_0/A:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742265_1441 (size=14541) 2024-11-19T05:26:14,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/84322771f600497dbf00367ba328fe37 2024-11-19T05:26:14,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/b1c6d297ef0846719d30524837c5e727 is 50, key is test_row_0/B:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742266_1442 (size=12151) 2024-11-19T05:26:14,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/b1c6d297ef0846719d30524837c5e727 2024-11-19T05:26:14,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/803ddb8032dd44729ec3004b3ed667af is 50, key is test_row_0/C:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742267_1443 (size=12151) 2024-11-19T05:26:14,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/803ddb8032dd44729ec3004b3ed667af 2024-11-19T05:26:14,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/84322771f600497dbf00367ba328fe37 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37 2024-11-19T05:26:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994034358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994034358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,365 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37, entries=200, sequenceid=154, filesize=14.2 K 2024-11-19T05:26:14,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994034360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/b1c6d297ef0846719d30524837c5e727 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727 2024-11-19T05:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994034367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994034378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,378 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,379 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T05:26:14,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/803ddb8032dd44729ec3004b3ed667af as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af 2024-11-19T05:26:14,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af, entries=150, sequenceid=154, filesize=11.9 K 2024-11-19T05:26:14,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c8584c7fbd7ef0113dc3083d3d909de9 in 106ms, sequenceid=154, compaction requested=true 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:14,395 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:14,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:26:14,395 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:14,396 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:14,396 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:14,396 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:14,396 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,396 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:14,396 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,396 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9de59f6272d645cfafde246b2984fb5e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=37.9 K 2024-11-19T05:26:14,396 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/96b92f467de4480ba3e075f03af53791, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=35.5 K 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de59f6272d645cfafde246b2984fb5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972860 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b92f467de4480ba3e075f03af53791, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972860 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ae54bdc65c499f92420d67e8550563, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731993973041 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c4db25aad2264c5f82ee2c4903d113e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731993973041 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 84322771f600497dbf00367ba328fe37, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:14,397 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c6d297ef0846719d30524837c5e727, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:14,423 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:14,424 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/896e0d4bbc1f4f77afdda5765e4e599c is 50, key is test_row_0/A:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,442 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:14,443 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/6e1352dd86cb459f815bdd1ae5aa009b is 50, key is test_row_0/B:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:14,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:14,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:26:14,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742268_1444 (size=12493) 2024-11-19T05:26:14,488 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/896e0d4bbc1f4f77afdda5765e4e599c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/896e0d4bbc1f4f77afdda5765e4e599c 2024-11-19T05:26:14,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/204114e1d0354da9acd292b83ddb0b8b is 50, key is test_row_0/A:col10/1731993974480/Put/seqid=0 2024-11-19T05:26:14,493 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into 896e0d4bbc1f4f77afdda5765e4e599c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:14,493 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:14,493 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=13, startTime=1731993974395; duration=0sec 2024-11-19T05:26:14,493 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:14,493 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:14,493 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:14,494 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:14,494 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:14,494 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,494 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5f4c86f3607a4456b055456a2b055dc3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=35.5 K 2024-11-19T05:26:14,494 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4c86f3607a4456b055456a2b055dc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731993972860 2024-11-19T05:26:14,494 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f059dae50440bd9296af43c0d5e78a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731993973041 2024-11-19T05:26:14,495 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 803ddb8032dd44729ec3004b3ed667af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:14,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742269_1445 (size=12493) 2024-11-19T05:26:14,507 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/6e1352dd86cb459f815bdd1ae5aa009b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6e1352dd86cb459f815bdd1ae5aa009b 2024-11-19T05:26:14,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742270_1446 (size=12147) 2024-11-19T05:26:14,514 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 6e1352dd86cb459f815bdd1ae5aa009b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:14,514 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:14,514 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=13, startTime=1731993974395; duration=0sec 2024-11-19T05:26:14,514 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:14,514 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:14,514 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:14,514 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36a3531fe15e4ac6844ea31e410fceac is 50, key is test_row_0/C:col10/1731993973929/Put/seqid=0 2024-11-19T05:26:14,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742271_1447 (size=12493) 2024-11-19T05:26:14,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994034630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-19T05:26:14,727 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:14,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-19T05:26:14,730 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:14,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:14,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994034747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:14,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T05:26:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:14,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:14,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:14,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/204114e1d0354da9acd292b83ddb0b8b 2024-11-19T05:26:14,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e3c2e0d127b54312a6729b56a7e02fce is 50, key is test_row_0/B:col10/1731993974480/Put/seqid=0 2024-11-19T05:26:14,925 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36a3531fe15e4ac6844ea31e410fceac as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36a3531fe15e4ac6844ea31e410fceac 2024-11-19T05:26:14,932 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 36a3531fe15e4ac6844ea31e410fceac(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:14,933 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:14,933 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=13, startTime=1731993974395; duration=0sec 2024-11-19T05:26:14,933 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:14,933 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:14,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742272_1448 (size=9757) 2024-11-19T05:26:14,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e3c2e0d127b54312a6729b56a7e02fce 2024-11-19T05:26:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994034957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:14,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e9193a8004d9469e9806fa2e5f0a3471 is 50, key is test_row_0/C:col10/1731993974480/Put/seqid=0 2024-11-19T05:26:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742273_1449 (size=9757) 2024-11-19T05:26:14,995 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e9193a8004d9469e9806fa2e5f0a3471 2024-11-19T05:26:15,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/204114e1d0354da9acd292b83ddb0b8b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b 2024-11-19T05:26:15,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b, entries=150, sequenceid=166, filesize=11.9 K 2024-11-19T05:26:15,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e3c2e0d127b54312a6729b56a7e02fce as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce 2024-11-19T05:26:15,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce, entries=100, sequenceid=166, filesize=9.5 K 2024-11-19T05:26:15,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e9193a8004d9469e9806fa2e5f0a3471 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471 2024-11-19T05:26:15,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471, entries=100, sequenceid=166, filesize=9.5 K 2024-11-19T05:26:15,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for c8584c7fbd7ef0113dc3083d3d909de9 in 549ms, sequenceid=166, compaction requested=false 2024-11-19T05:26:15,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:15,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-19T05:26:15,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:15,041 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-19T05:26:15,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/333cba1b7c3f43b5ac93e80bf8a1434e is 50, key is test_row_0/A:col10/1731993974614/Put/seqid=0 2024-11-19T05:26:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:15,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742274_1450 (size=12151) 2024-11-19T05:26:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:15,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:15,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994035341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:15,465 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/333cba1b7c3f43b5ac93e80bf8a1434e 2024-11-19T05:26:15,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994035458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:15,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/39fd872cc760484181c269beb0bff3f8 is 50, key is test_row_0/B:col10/1731993974614/Put/seqid=0 2024-11-19T05:26:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742275_1451 (size=12151) 2024-11-19T05:26:15,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994035677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:15,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:15,898 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/39fd872cc760484181c269beb0bff3f8 2024-11-19T05:26:15,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/d19a7d577b4a4ace8271c458175deb09 is 50, key is test_row_0/C:col10/1731993974614/Put/seqid=0 2024-11-19T05:26:15,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742276_1452 (size=12151) 2024-11-19T05:26:15,924 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/d19a7d577b4a4ace8271c458175deb09 2024-11-19T05:26:15,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/333cba1b7c3f43b5ac93e80bf8a1434e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e 2024-11-19T05:26:15,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T05:26:15,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/39fd872cc760484181c269beb0bff3f8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8 2024-11-19T05:26:15,942 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T05:26:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/d19a7d577b4a4ace8271c458175deb09 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09 2024-11-19T05:26:15,946 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09, entries=150, sequenceid=193, filesize=11.9 K 2024-11-19T05:26:15,946 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for c8584c7fbd7ef0113dc3083d3d909de9 in 906ms, sequenceid=193, compaction requested=true 2024-11-19T05:26:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:15,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-19T05:26:15,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-19T05:26:15,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-19T05:26:15,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2160 sec 2024-11-19T05:26:15,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2220 sec 2024-11-19T05:26:16,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:16,002 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:16,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:26:16,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/e77d38edc0cb4cd49b93cc7716166e89 is 50, key is test_row_0/A:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742277_1453 (size=14541) 2024-11-19T05:26:16,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:16,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994036195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:16,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:16,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994036317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:16,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/e77d38edc0cb4cd49b93cc7716166e89 2024-11-19T05:26:16,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/6bcd9cb520bd413aa0d886dd8807c213 is 50, key is test_row_0/B:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742278_1454 (size=12151) 2024-11-19T05:26:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994036537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:16,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/6bcd9cb520bd413aa0d886dd8807c213 2024-11-19T05:26:16,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/2303ba352af54510ae0e74790ec31f56 is 50, key is test_row_0/C:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:16,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994036847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:16,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:16,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742279_1455 (size=12151) 2024-11-19T05:26:16,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/2303ba352af54510ae0e74790ec31f56 2024-11-19T05:26:16,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/e77d38edc0cb4cd49b93cc7716166e89 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89 2024-11-19T05:26:16,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89, entries=200, sequenceid=206, filesize=14.2 K 2024-11-19T05:26:16,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/6bcd9cb520bd413aa0d886dd8807c213 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213 2024-11-19T05:26:16,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T05:26:16,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/2303ba352af54510ae0e74790ec31f56 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56 2024-11-19T05:26:16,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56, entries=150, sequenceid=206, filesize=11.9 K 2024-11-19T05:26:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-19T05:26:16,877 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:16,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for c8584c7fbd7ef0113dc3083d3d909de9 in 875ms, sequenceid=206, compaction requested=true 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:16,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:16,878 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:26:16,878 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-19T05:26:16,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:16,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T05:26:16,880 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:16,880 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:16,881 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:16,881 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:16,881 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:16,881 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/896e0d4bbc1f4f77afdda5765e4e599c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=50.1 K 2024-11-19T05:26:16,881 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:16,881 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 896e0d4bbc1f4f77afdda5765e4e599c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:16,881 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:16,881 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:16,881 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6e1352dd86cb459f815bdd1ae5aa009b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=45.5 K 2024-11-19T05:26:16,881 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 204114e1d0354da9acd292b83ddb0b8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731993974302 2024-11-19T05:26:16,882 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e1352dd86cb459f815bdd1ae5aa009b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:16,882 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 333cba1b7c3f43b5ac93e80bf8a1434e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731993974614 2024-11-19T05:26:16,882 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c2e0d127b54312a6729b56a7e02fce, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731993974351 2024-11-19T05:26:16,882 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e77d38edc0cb4cd49b93cc7716166e89, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975326 2024-11-19T05:26:16,882 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39fd872cc760484181c269beb0bff3f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731993974614 2024-11-19T05:26:16,883 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bcd9cb520bd413aa0d886dd8807c213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975336 2024-11-19T05:26:16,896 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:16,896 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:16,896 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/e8c047278c1f49f688819615151bd53b is 50, key is test_row_0/A:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,897 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/d37742f2601443adb07366f7a3c9a4b9 is 50, key is test_row_0/B:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742281_1457 (size=12629) 2024-11-19T05:26:16,907 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/d37742f2601443adb07366f7a3c9a4b9 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d37742f2601443adb07366f7a3c9a4b9 2024-11-19T05:26:16,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742280_1456 (size=12629) 2024-11-19T05:26:16,914 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into d37742f2601443adb07366f7a3c9a4b9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:16,914 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:16,914 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=12, startTime=1731993976878; duration=0sec 2024-11-19T05:26:16,914 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:16,914 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:16,915 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:16,916 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:16,916 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:16,916 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:16,917 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/e8c047278c1f49f688819615151bd53b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e8c047278c1f49f688819615151bd53b 2024-11-19T05:26:16,917 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36a3531fe15e4ac6844ea31e410fceac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=45.5 K 2024-11-19T05:26:16,917 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a3531fe15e4ac6844ea31e410fceac, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731993973882 2024-11-19T05:26:16,917 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9193a8004d9469e9806fa2e5f0a3471, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731993974351 2024-11-19T05:26:16,917 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting d19a7d577b4a4ace8271c458175deb09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731993974614 2024-11-19T05:26:16,918 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2303ba352af54510ae0e74790ec31f56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975336 2024-11-19T05:26:16,920 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into e8c047278c1f49f688819615151bd53b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:16,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:16,921 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=12, startTime=1731993976878; duration=0sec 2024-11-19T05:26:16,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:16,921 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:16,927 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:16,928 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/9e054f9be706461ea8a76455901bd0f5 is 50, key is test_row_0/C:col10/1731993976001/Put/seqid=0 2024-11-19T05:26:16,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742282_1458 (size=12629) 2024-11-19T05:26:16,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T05:26:17,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-19T05:26:17,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:17,033 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:26:17,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0062aaa511db4d3cbdfa14e3cac43b9a is 50, key is test_row_0/A:col10/1731993976141/Put/seqid=0 2024-11-19T05:26:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742283_1459 (size=12151) 2024-11-19T05:26:17,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T05:26:17,346 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/9e054f9be706461ea8a76455901bd0f5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/9e054f9be706461ea8a76455901bd0f5 2024-11-19T05:26:17,351 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 9e054f9be706461ea8a76455901bd0f5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:17,351 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:17,351 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=12, startTime=1731993976878; duration=0sec 2024-11-19T05:26:17,351 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:17,351 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:17,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:17,441 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0062aaa511db4d3cbdfa14e3cac43b9a 2024-11-19T05:26:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994037436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,449 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:17,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e35c93bd8c6a433aaa904b03461b7b02 is 50, key is test_row_0/B:col10/1731993976141/Put/seqid=0 2024-11-19T05:26:17,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742284_1460 (size=12151) 2024-11-19T05:26:17,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e35c93bd8c6a433aaa904b03461b7b02 2024-11-19T05:26:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T05:26:17,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5aedaccff57640bea147d4d5d4399ec4 is 50, key is test_row_0/C:col10/1731993976141/Put/seqid=0 2024-11-19T05:26:17,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742285_1461 (size=12151) 2024-11-19T05:26:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994037557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,564 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:17,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:17,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994037767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:17,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:17,925 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5aedaccff57640bea147d4d5d4399ec4 2024-11-19T05:26:17,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0062aaa511db4d3cbdfa14e3cac43b9a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a 2024-11-19T05:26:17,935 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T05:26:17,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e35c93bd8c6a433aaa904b03461b7b02 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02 2024-11-19T05:26:17,939 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T05:26:17,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/5aedaccff57640bea147d4d5d4399ec4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4 2024-11-19T05:26:17,944 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4, entries=150, sequenceid=231, filesize=11.9 K 2024-11-19T05:26:17,945 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for c8584c7fbd7ef0113dc3083d3d909de9 in 911ms, sequenceid=231, compaction requested=false 2024-11-19T05:26:17,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:17,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:17,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-19T05:26:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-19T05:26:17,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-19T05:26:17,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0650 sec 2024-11-19T05:26:17,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.0720 sec 2024-11-19T05:26:18,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-19T05:26:18,017 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:18,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-19T05:26:18,019 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:18,020 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:18,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:18,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:18,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:26:18,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c619c695e3a045d29dfe55825001bedd is 50, key is test_row_0/A:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:18,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742286_1462 (size=14541) 2024-11-19T05:26:18,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:18,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994038241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:18,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994038357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45530 deadline: 1731994038377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,382 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45484 deadline: 1731994038398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,406 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:18,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45482 deadline: 1731994038398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,410 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:18,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45532 deadline: 1731994038417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,424 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in c8584c7fbd7ef0113dc3083d3d909de9 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c619c695e3a045d29dfe55825001bedd 2024-11-19T05:26:18,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e89cfd55ce8147bf82d5073ea2f2f1c7 is 50, key is test_row_0/B:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:18,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742287_1463 (size=12151) 2024-11-19T05:26:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994038577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:18,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e89cfd55ce8147bf82d5073ea2f2f1c7 2024-11-19T05:26:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994038899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:18,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36ce3c04d8cc44b89cbc72c4cb49800f is 50, key is test_row_0/C:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:18,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742288_1464 (size=12151) 2024-11-19T05:26:18,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:18,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:19,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:19,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:19,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:19,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:19,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36ce3c04d8cc44b89cbc72c4cb49800f 2024-11-19T05:26:19,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/c619c695e3a045d29dfe55825001bedd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd 2024-11-19T05:26:19,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd, entries=200, sequenceid=246, filesize=14.2 K 2024-11-19T05:26:19,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e89cfd55ce8147bf82d5073ea2f2f1c7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7 2024-11-19T05:26:19,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T05:26:19,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36ce3c04d8cc44b89cbc72c4cb49800f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f 2024-11-19T05:26:19,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f, entries=150, sequenceid=246, filesize=11.9 K 2024-11-19T05:26:19,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for c8584c7fbd7ef0113dc3083d3d909de9 in 1254ms, sequenceid=246, compaction requested=true 2024-11-19T05:26:19,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:19,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:19,334 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:19,334 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:19,334 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:19,335 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:19,335 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:19,335 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,335 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d37742f2601443adb07366f7a3c9a4b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.1 K 2024-11-19T05:26:19,335 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:19,336 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:19,336 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,336 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e8c047278c1f49f688819615151bd53b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=38.4 K 2024-11-19T05:26:19,336 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d37742f2601443adb07366f7a3c9a4b9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975336 2024-11-19T05:26:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:19,336 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c047278c1f49f688819615151bd53b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975336 2024-11-19T05:26:19,336 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e35c93bd8c6a433aaa904b03461b7b02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731993976141 2024-11-19T05:26:19,336 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0062aaa511db4d3cbdfa14e3cac43b9a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731993976141 2024-11-19T05:26:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:19,337 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e89cfd55ce8147bf82d5073ea2f2f1c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:19,337 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c619c695e3a045d29dfe55825001bedd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:19,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:19,344 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:19,344 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/a183d59b3fba4aaa8b589a311a724021 is 50, key is test_row_0/A:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:19,346 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:19,346 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/29028fbe4f3840029331f08fad764c38 is 50, key is test_row_0/B:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742290_1466 (size=12731) 2024-11-19T05:26:19,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742289_1465 (size=12731) 2024-11-19T05:26:19,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-19T05:26:19,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,395 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:26:19,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/574ef46607cf439d90766f274fd99046 is 50, key is test_row_0/A:col10/1731993978225/Put/seqid=0 2024-11-19T05:26:19,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742291_1467 (size=12301) 2024-11-19T05:26:19,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:19,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:19,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:19,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994039519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:19,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994039637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:19,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:19,762 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/29028fbe4f3840029331f08fad764c38 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/29028fbe4f3840029331f08fad764c38 2024-11-19T05:26:19,765 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/a183d59b3fba4aaa8b589a311a724021 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a183d59b3fba4aaa8b589a311a724021 2024-11-19T05:26:19,766 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 29028fbe4f3840029331f08fad764c38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:19,766 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:19,766 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=13, startTime=1731993979334; duration=0sec 2024-11-19T05:26:19,766 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:19,766 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:19,766 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:19,767 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:19,767 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:19,767 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:19,767 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/9e054f9be706461ea8a76455901bd0f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.1 K 2024-11-19T05:26:19,768 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e054f9be706461ea8a76455901bd0f5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731993975336 2024-11-19T05:26:19,768 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into a183d59b3fba4aaa8b589a311a724021(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:19,768 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:19,768 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=13, startTime=1731993979334; duration=0sec 2024-11-19T05:26:19,768 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:19,769 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:19,769 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aedaccff57640bea147d4d5d4399ec4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731993976141 2024-11-19T05:26:19,769 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 36ce3c04d8cc44b89cbc72c4cb49800f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:19,775 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:19,775 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/35ffb965301943d991d708bbb9971206 is 50, key is test_row_0/C:col10/1731993977410/Put/seqid=0 2024-11-19T05:26:19,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742292_1468 (size=12731) 2024-11-19T05:26:19,783 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/35ffb965301943d991d708bbb9971206 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/35ffb965301943d991d708bbb9971206 2024-11-19T05:26:19,786 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 35ffb965301943d991d708bbb9971206(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:19,786 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:19,786 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=13, startTime=1731993979336; duration=0sec 2024-11-19T05:26:19,786 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:19,786 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:19,804 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/574ef46607cf439d90766f274fd99046 2024-11-19T05:26:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/aeaff6d456f540109b3e2473a2245efd is 50, key is test_row_0/B:col10/1731993978225/Put/seqid=0 2024-11-19T05:26:19,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742293_1469 (size=12301) 2024-11-19T05:26:19,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994039857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:20,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994040167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:20,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:20,213 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/aeaff6d456f540109b3e2473a2245efd 2024-11-19T05:26:20,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6174006de5d14c4b9fdf096780ecd8b3 is 50, key is test_row_0/C:col10/1731993978225/Put/seqid=0 2024-11-19T05:26:20,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742294_1470 (size=12301) 2024-11-19T05:26:20,621 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6174006de5d14c4b9fdf096780ecd8b3 2024-11-19T05:26:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/574ef46607cf439d90766f274fd99046 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046 2024-11-19T05:26:20,629 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T05:26:20,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/aeaff6d456f540109b3e2473a2245efd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd 2024-11-19T05:26:20,633 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T05:26:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6174006de5d14c4b9fdf096780ecd8b3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3 2024-11-19T05:26:20,637 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3, entries=150, sequenceid=268, filesize=12.0 K 2024-11-19T05:26:20,638 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c8584c7fbd7ef0113dc3083d3d909de9 in 1243ms, sequenceid=268, compaction requested=false 2024-11-19T05:26:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:20,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-19T05:26:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-19T05:26:20,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-19T05:26:20,641 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-11-19T05:26:20,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6230 sec 2024-11-19T05:26:20,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:20,694 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:20,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:26:20,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0a88b20261a946398751d80990420f4a is 50, key is test_row_0/A:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:20,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742295_1471 (size=14741) 2024-11-19T05:26:20,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0a88b20261a946398751d80990420f4a 2024-11-19T05:26:20,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/033e9095ad8745e6a082f93cd4c82073 is 50, key is test_row_0/B:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:20,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742296_1472 (size=12301) 2024-11-19T05:26:20,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994040846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:20,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:20,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994040957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:20,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:21,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/033e9095ad8745e6a082f93cd4c82073 2024-11-19T05:26:21,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0cb95efeec02441fa76a27b899ea7c1e is 50, key is test_row_0/C:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:21,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742297_1473 (size=12301) 2024-11-19T05:26:21,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0cb95efeec02441fa76a27b899ea7c1e 2024-11-19T05:26:21,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0a88b20261a946398751d80990420f4a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a 2024-11-19T05:26:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a, entries=200, sequenceid=286, filesize=14.4 K 2024-11-19T05:26:21,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/033e9095ad8745e6a082f93cd4c82073 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073 2024-11-19T05:26:21,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T05:26:21,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0cb95efeec02441fa76a27b899ea7c1e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e 2024-11-19T05:26:21,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994041167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:21,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e, entries=150, sequenceid=286, filesize=12.0 K 2024-11-19T05:26:21,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for c8584c7fbd7ef0113dc3083d3d909de9 in 483ms, sequenceid=286, compaction requested=true 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:21,177 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:26:21,179 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:21,179 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:21,179 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:21,180 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a183d59b3fba4aaa8b589a311a724021, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=38.8 K 2024-11-19T05:26:21,180 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:21,180 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a183d59b3fba4aaa8b589a311a724021, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:21,180 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 574ef46607cf439d90766f274fd99046, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731993978215 2024-11-19T05:26:21,181 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a88b20261a946398751d80990420f4a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979464 2024-11-19T05:26:21,181 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:21,181 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:21,181 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:21,181 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/29028fbe4f3840029331f08fad764c38, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.5 K 2024-11-19T05:26:21,184 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 29028fbe4f3840029331f08fad764c38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:21,185 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting aeaff6d456f540109b3e2473a2245efd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731993978215 2024-11-19T05:26:21,186 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 033e9095ad8745e6a082f93cd4c82073, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979490 2024-11-19T05:26:21,206 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:21,206 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7d6c947430154569bb5c0402d6f5b1e0 is 50, key is test_row_0/A:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:21,209 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:21,210 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/ba63371193bf4804ac8f35b29b0802ab is 50, key is test_row_0/B:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:21,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742298_1474 (size=12983) 2024-11-19T05:26:21,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742299_1475 (size=12983) 2024-11-19T05:26:21,237 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/ba63371193bf4804ac8f35b29b0802ab as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ba63371193bf4804ac8f35b29b0802ab 2024-11-19T05:26:21,242 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into ba63371193bf4804ac8f35b29b0802ab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:21,242 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:21,242 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=13, startTime=1731993981177; duration=0sec 2024-11-19T05:26:21,242 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:21,242 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:21,242 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:21,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:21,244 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:21,244 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:21,244 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/35ffb965301943d991d708bbb9971206, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.5 K 2024-11-19T05:26:21,246 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ffb965301943d991d708bbb9971206, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731993977410 2024-11-19T05:26:21,247 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6174006de5d14c4b9fdf096780ecd8b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731993978215 2024-11-19T05:26:21,247 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cb95efeec02441fa76a27b899ea7c1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979490 2024-11-19T05:26:21,265 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:21,266 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0fe99a8ebf07481a8af25b0c627003e7 is 50, key is test_row_0/C:col10/1731993980693/Put/seqid=0 2024-11-19T05:26:21,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742300_1476 (size=12983) 2024-11-19T05:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:21,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:21,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-19T05:26:21,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/9ca51d768eff40fa8ca272705f91433d is 50, key is test_row_0/A:col10/1731993980767/Put/seqid=0 2024-11-19T05:26:21,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742301_1477 (size=14741) 2024-11-19T05:26:21,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994041601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:21,618 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7d6c947430154569bb5c0402d6f5b1e0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7d6c947430154569bb5c0402d6f5b1e0 2024-11-19T05:26:21,622 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into 7d6c947430154569bb5c0402d6f5b1e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:21,622 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:21,622 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=13, startTime=1731993981177; duration=0sec 2024-11-19T05:26:21,622 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:21,622 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:21,680 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0fe99a8ebf07481a8af25b0c627003e7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0fe99a8ebf07481a8af25b0c627003e7 2024-11-19T05:26:21,686 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 0fe99a8ebf07481a8af25b0c627003e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:21,686 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:21,686 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=13, startTime=1731993981177; duration=0sec 2024-11-19T05:26:21,686 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:21,686 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:21,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994041717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:21,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/9ca51d768eff40fa8ca272705f91433d 2024-11-19T05:26:21,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/2ccd6d2de1904d03acb77cdf908d2c26 is 50, key is test_row_0/B:col10/1731993980767/Put/seqid=0 2024-11-19T05:26:21,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742302_1478 (size=12301) 2024-11-19T05:26:21,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/2ccd6d2de1904d03acb77cdf908d2c26 2024-11-19T05:26:21,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6df6a7b9cecd40a7983fcb8f66e10b81 is 50, key is test_row_0/C:col10/1731993980767/Put/seqid=0 2024-11-19T05:26:21,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994041937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:21,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742303_1479 (size=12301) 2024-11-19T05:26:21,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6df6a7b9cecd40a7983fcb8f66e10b81 2024-11-19T05:26:21,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/9ca51d768eff40fa8ca272705f91433d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d 2024-11-19T05:26:21,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d, entries=200, sequenceid=309, filesize=14.4 K 2024-11-19T05:26:21,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/2ccd6d2de1904d03acb77cdf908d2c26 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26 2024-11-19T05:26:21,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T05:26:21,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/6df6a7b9cecd40a7983fcb8f66e10b81 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81 2024-11-19T05:26:22,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81, entries=150, sequenceid=309, filesize=12.0 K 2024-11-19T05:26:22,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for c8584c7fbd7ef0113dc3083d3d909de9 in 523ms, sequenceid=309, compaction requested=false 2024-11-19T05:26:22,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-19T05:26:22,187 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:22,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:22,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-19T05:26:22,191 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:22,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:22,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:22,264 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:22,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-19T05:26:22,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/f5a2cb8eae4a4837a32f79b62eda601a is 50, key is test_row_0/A:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742304_1480 (size=14741) 2024-11-19T05:26:22,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/f5a2cb8eae4a4837a32f79b62eda601a 2024-11-19T05:26:22,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/34a4c621a23a46f78b6d33cffc6cc1e1 is 50, key is test_row_0/B:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:22,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742305_1481 (size=12301) 2024-11-19T05:26:22,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/34a4c621a23a46f78b6d33cffc6cc1e1 2024-11-19T05:26:22,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/239b47546e7144db917f0b06b606068d is 50, key is test_row_0/C:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:22,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742306_1482 (size=12301) 2024-11-19T05:26:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/239b47546e7144db917f0b06b606068d 2024-11-19T05:26:22,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/f5a2cb8eae4a4837a32f79b62eda601a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a 2024-11-19T05:26:22,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a, entries=200, sequenceid=326, filesize=14.4 K 2024-11-19T05:26:22,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/34a4c621a23a46f78b6d33cffc6cc1e1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1 2024-11-19T05:26:22,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T05:26:22,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/239b47546e7144db917f0b06b606068d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d 2024-11-19T05:26:22,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d, entries=150, sequenceid=326, filesize=12.0 K 2024-11-19T05:26:22,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=46.96 KB/48090 for c8584c7fbd7ef0113dc3083d3d909de9 in 116ms, sequenceid=326, compaction requested=true 2024-11-19T05:26:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:22,380 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:22,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:22,381 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:22,381 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:22,381 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,381 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7d6c947430154569bb5c0402d6f5b1e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=41.5 K 2024-11-19T05:26:22,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:22,382 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d6c947430154569bb5c0402d6f5b1e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979490 2024-11-19T05:26:22,382 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:22,382 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ca51d768eff40fa8ca272705f91433d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993980767 2024-11-19T05:26:22,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:22,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:22,384 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a2cb8eae4a4837a32f79b62eda601a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981552 2024-11-19T05:26:22,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:22,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:22,385 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,385 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ba63371193bf4804ac8f35b29b0802ab, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.7 K 2024-11-19T05:26:22,385 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ba63371193bf4804ac8f35b29b0802ab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979490 2024-11-19T05:26:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:22,387 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:22,388 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ccd6d2de1904d03acb77cdf908d2c26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993980767 2024-11-19T05:26:22,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:22,388 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 34a4c621a23a46f78b6d33cffc6cc1e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981580 2024-11-19T05:26:22,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7993efba5b4c4e32823677ba899d2c2c is 50, key is test_row_0/A:col10/1731993982367/Put/seqid=0 2024-11-19T05:26:22,418 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:22,418 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/6f0de4ba0f374f47936d557ae70b7faa is 50, key is test_row_0/A:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,427 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:22,427 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/31f7821621984862a9c3f2165ecbbc77 is 50, key is test_row_0/B:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742307_1483 (size=14741) 2024-11-19T05:26:22,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7993efba5b4c4e32823677ba899d2c2c 2024-11-19T05:26:22,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742308_1484 (size=13085) 2024-11-19T05:26:22,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742309_1485 (size=13085) 2024-11-19T05:26:22,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e938df5d42b54c4389bfd08a6d810f88 is 50, key is test_row_0/B:col10/1731993982367/Put/seqid=0 2024-11-19T05:26:22,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:22,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:22,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742310_1486 (size=12301) 2024-11-19T05:26:22,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994042669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:22,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:22,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994042787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:22,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:22,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:22,896 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/6f0de4ba0f374f47936d557ae70b7faa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6f0de4ba0f374f47936d557ae70b7faa 2024-11-19T05:26:22,899 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/31f7821621984862a9c3f2165ecbbc77 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/31f7821621984862a9c3f2165ecbbc77 2024-11-19T05:26:22,900 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into 6f0de4ba0f374f47936d557ae70b7faa(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:22,900 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:22,900 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=13, startTime=1731993982380; duration=0sec 2024-11-19T05:26:22,900 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:22,900 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:22,901 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:22,902 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:22,902 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:22,902 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,902 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0fe99a8ebf07481a8af25b0c627003e7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=36.7 K 2024-11-19T05:26:22,904 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe99a8ebf07481a8af25b0c627003e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731993979490 2024-11-19T05:26:22,905 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df6a7b9cecd40a7983fcb8f66e10b81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731993980767 2024-11-19T05:26:22,905 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239b47546e7144db917f0b06b606068d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981580 2024-11-19T05:26:22,909 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 31f7821621984862a9c3f2165ecbbc77(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:22,909 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:22,909 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=13, startTime=1731993982382; duration=0sec 2024-11-19T05:26:22,909 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:22,909 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:22,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e938df5d42b54c4389bfd08a6d810f88 2024-11-19T05:26:22,922 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:22,922 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36087400bf06452fb88d3b788d07baea is 50, key is test_row_0/C:col10/1731993982259/Put/seqid=0 2024-11-19T05:26:22,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/cb2107f4c7734838b9379e644b7e814b is 50, key is test_row_0/C:col10/1731993982367/Put/seqid=0 2024-11-19T05:26:22,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742311_1487 (size=13085) 2024-11-19T05:26:22,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742312_1488 (size=12301) 2024-11-19T05:26:22,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:22,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994043007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:23,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:23,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:23,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994043317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:23,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:23,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:23,333 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/36087400bf06452fb88d3b788d07baea as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36087400bf06452fb88d3b788d07baea 2024-11-19T05:26:23,337 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 36087400bf06452fb88d3b788d07baea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:23,338 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:23,338 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=13, startTime=1731993982383; duration=0sec 2024-11-19T05:26:23,338 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:23,338 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:23,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/cb2107f4c7734838b9379e644b7e814b 2024-11-19T05:26:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/7993efba5b4c4e32823677ba899d2c2c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c 2024-11-19T05:26:23,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c, entries=200, sequenceid=337, filesize=14.4 K 2024-11-19T05:26:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/e938df5d42b54c4389bfd08a6d810f88 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88 2024-11-19T05:26:23,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88, entries=150, sequenceid=337, filesize=12.0 K 2024-11-19T05:26:23,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/cb2107f4c7734838b9379e644b7e814b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b 2024-11-19T05:26:23,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b, entries=150, sequenceid=337, filesize=12.0 K 2024-11-19T05:26:23,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c8584c7fbd7ef0113dc3083d3d909de9 in 972ms, sequenceid=337, compaction requested=false 2024-11-19T05:26:23,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:23,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-19T05:26:23,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,429 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:23,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/4d3b2f424d304ef59233ae23a0048ba7 is 50, key is test_row_0/A:col10/1731993982617/Put/seqid=0 2024-11-19T05:26:23,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742313_1489 (size=12301) 2024-11-19T05:26:23,436 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/4d3b2f424d304ef59233ae23a0048ba7 2024-11-19T05:26:23,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/329514a941b94191b9f6deba622368e0 is 50, key is test_row_0/B:col10/1731993982617/Put/seqid=0 2024-11-19T05:26:23,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742314_1490 (size=12301) 2024-11-19T05:26:23,455 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/329514a941b94191b9f6deba622368e0 2024-11-19T05:26:23,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f751963210534a7896c258cba83fa6aa is 50, key is test_row_0/C:col10/1731993982617/Put/seqid=0 2024-11-19T05:26:23,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742315_1491 (size=12301) 2024-11-19T05:26:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:23,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:23,872 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f751963210534a7896c258cba83fa6aa 2024-11-19T05:26:23,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/4d3b2f424d304ef59233ae23a0048ba7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7 2024-11-19T05:26:23,886 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7, entries=150, sequenceid=365, filesize=12.0 K 2024-11-19T05:26:23,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/329514a941b94191b9f6deba622368e0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0 2024-11-19T05:26:23,894 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0, entries=150, sequenceid=365, filesize=12.0 K 2024-11-19T05:26:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f751963210534a7896c258cba83fa6aa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa 2024-11-19T05:26:23,900 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa, entries=150, sequenceid=365, filesize=12.0 K 2024-11-19T05:26:23,901 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=40.25 KB/41220 for c8584c7fbd7ef0113dc3083d3d909de9 in 473ms, sequenceid=365, compaction requested=true 2024-11-19T05:26:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:23,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-19T05:26:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-19T05:26:23,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-19T05:26:23,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7100 sec 2024-11-19T05:26:23,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7250 sec 2024-11-19T05:26:23,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:23,931 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:23,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:23,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/90528aa734b541b8baefb7a06d8a55b4 is 50, key is test_row_0/A:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:23,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742316_1492 (size=14741) 2024-11-19T05:26:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 312 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994044149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 313 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994044267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-19T05:26:24,338 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:24,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-19T05:26:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:24,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:24,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:24,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:24,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/90528aa734b541b8baefb7a06d8a55b4 2024-11-19T05:26:24,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/82158480b4ed439981809150b8f6127e is 50, key is test_row_0/B:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:24,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742317_1493 (size=12301) 2024-11-19T05:26:24,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/82158480b4ed439981809150b8f6127e 2024-11-19T05:26:24,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/87f5d58abaf44e78a9bd2b256ee70c06 is 50, key is test_row_0/C:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:24,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742318_1494 (size=12301) 2024-11-19T05:26:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:24,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 314 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994044477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:24,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T05:26:24,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:24,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T05:26:24,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:24,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:24,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/87f5d58abaf44e78a9bd2b256ee70c06 2024-11-19T05:26:24,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/90528aa734b541b8baefb7a06d8a55b4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4 2024-11-19T05:26:24,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4, entries=200, sequenceid=376, filesize=14.4 K 2024-11-19T05:26:24,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/82158480b4ed439981809150b8f6127e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e 2024-11-19T05:26:24,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e, entries=150, sequenceid=376, filesize=12.0 K 2024-11-19T05:26:24,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/87f5d58abaf44e78a9bd2b256ee70c06 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06 2024-11-19T05:26:24,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06, entries=150, sequenceid=376, filesize=12.0 K 2024-11-19T05:26:24,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for c8584c7fbd7ef0113dc3083d3d909de9 in 847ms, sequenceid=376, compaction requested=true 2024-11-19T05:26:24,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:24,778 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54868 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:24,779 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,779 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6f0de4ba0f374f47936d557ae70b7faa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=53.6 K 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0de4ba0f374f47936d557ae70b7faa, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981580 2024-11-19T05:26:24,779 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:24,779 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,779 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/31f7821621984862a9c3f2165ecbbc77, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=48.8 K 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7993efba5b4c4e32823677ba899d2c2c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731993982332 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f7821621984862a9c3f2165ecbbc77, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981580 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d3b2f424d304ef59233ae23a0048ba7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731993982617 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting e938df5d42b54c4389bfd08a6d810f88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731993982332 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90528aa734b541b8baefb7a06d8a55b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731993983876 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 329514a941b94191b9f6deba622368e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731993982617 2024-11-19T05:26:24,780 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 82158480b4ed439981809150b8f6127e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731993983890 2024-11-19T05:26:24,786 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:24,787 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:24,787 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/bc7330a238d64a8f85c0a5626cdf66d2 is 50, key is test_row_0/B:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:24,787 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/fa2620346d4c427983c3759d20081380 is 50, key is test_row_0/A:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:24,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742320_1496 (size=13221) 2024-11-19T05:26:24,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742319_1495 (size=13221) 2024-11-19T05:26:24,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-19T05:26:24,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:24,798 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:24,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/8fef224affb44d64b17ba491e7e0b29b is 50, key is test_row_0/A:col10/1731993984147/Put/seqid=0 2024-11-19T05:26:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:24,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742321_1497 (size=12301) 2024-11-19T05:26:24,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994044862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:24,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994044987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:24,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:25,199 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/fa2620346d4c427983c3759d20081380 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/fa2620346d4c427983c3759d20081380 2024-11-19T05:26:25,201 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/bc7330a238d64a8f85c0a5626cdf66d2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/bc7330a238d64a8f85c0a5626cdf66d2 2024-11-19T05:26:25,203 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into fa2620346d4c427983c3759d20081380(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:25,203 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:25,203 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=12, startTime=1731993984778; duration=0sec 2024-11-19T05:26:25,203 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:25,203 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:25,203 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:25,204 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:25,204 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/C is initiating minor compaction (all files) 2024-11-19T05:26:25,204 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/C in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:25,204 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36087400bf06452fb88d3b788d07baea, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=48.8 K 2024-11-19T05:26:25,205 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/8fef224affb44d64b17ba491e7e0b29b 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36087400bf06452fb88d3b788d07baea, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731993981580 2024-11-19T05:26:25,205 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into bc7330a238d64a8f85c0a5626cdf66d2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb2107f4c7734838b9379e644b7e814b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731993982332 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:25,205 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=12, startTime=1731993984779; duration=0sec 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f751963210534a7896c258cba83fa6aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731993982617 2024-11-19T05:26:25,205 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f5d58abaf44e78a9bd2b256ee70c06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731993983890 2024-11-19T05:26:25,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/4d73ed7142644d868c330d900403f0dd is 50, key is test_row_0/B:col10/1731993984147/Put/seqid=0 2024-11-19T05:26:25,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:25,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994045207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:25,214 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:25,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:25,215 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0c857d4ca46643a48667ad6be352c057 is 50, key is test_row_0/C:col10/1731993983890/Put/seqid=0 2024-11-19T05:26:25,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742322_1498 (size=13221) 2024-11-19T05:26:25,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742323_1499 (size=12301) 2024-11-19T05:26:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:25,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994045517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:25,624 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/0c857d4ca46643a48667ad6be352c057 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0c857d4ca46643a48667ad6be352c057 2024-11-19T05:26:25,626 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/4d73ed7142644d868c330d900403f0dd 2024-11-19T05:26:25,631 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/C of c8584c7fbd7ef0113dc3083d3d909de9 into 0c857d4ca46643a48667ad6be352c057(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:25,631 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:25,631 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/C, priority=12, startTime=1731993984779; duration=0sec 2024-11-19T05:26:25,631 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:25,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/a18140c7f8fb49e3a8069edc8bf674ae is 50, key is test_row_0/C:col10/1731993984147/Put/seqid=0 2024-11-19T05:26:25,636 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:25,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742324_1500 (size=12301) 2024-11-19T05:26:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45480 deadline: 1731994046027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 , the old value is region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., hostname=f990f635ffd0,38449,1731993838792, seqNum=2 because the exception is null or not the one we care about 2024-11-19T05:26:26,040 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/a18140c7f8fb49e3a8069edc8bf674ae 2024-11-19T05:26:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/8fef224affb44d64b17ba491e7e0b29b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b 2024-11-19T05:26:26,054 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T05:26:26,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/4d73ed7142644d868c330d900403f0dd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd 2024-11-19T05:26:26,060 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T05:26:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/a18140c7f8fb49e3a8069edc8bf674ae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/a18140c7f8fb49e3a8069edc8bf674ae 2024-11-19T05:26:26,066 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/a18140c7f8fb49e3a8069edc8bf674ae, entries=150, sequenceid=401, filesize=12.0 K 2024-11-19T05:26:26,069 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for c8584c7fbd7ef0113dc3083d3d909de9 in 1271ms, sequenceid=401, compaction requested=false 2024-11-19T05:26:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:26,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-19T05:26:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-19T05:26:26,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-19T05:26:26,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7330 sec 2024-11-19T05:26:26,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7420 sec 2024-11-19T05:26:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-19T05:26:26,488 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:26,489 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-19T05:26:26,492 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:26,493 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:26,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T05:26:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T05:26:26,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-19T05:26:26,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:26,660 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/a77468f27c734ae884f17e508700c75f is 50, key is test_row_0/A:col10/1731993984802/Put/seqid=0 2024-11-19T05:26:26,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742325_1501 (size=12301) 2024-11-19T05:26:26,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T05:26:27,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. as already flushing 2024-11-19T05:26:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:27,116 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:26:27,130 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/a77468f27c734ae884f17e508700c75f 2024-11-19T05:26:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T05:26:27,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/c14f46dc679141b99c6fc20c1f7b26fe is 50, key is test_row_0/B:col10/1731993984802/Put/seqid=0 2024-11-19T05:26:27,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742326_1502 (size=12301) 2024-11-19T05:26:27,177 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/c14f46dc679141b99c6fc20c1f7b26fe 2024-11-19T05:26:27,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f18bbba304e94e209c17870d6e827499 is 50, key is test_row_0/C:col10/1731993984802/Put/seqid=0 2024-11-19T05:26:27,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742327_1503 (size=12301) 2024-11-19T05:26:27,219 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f18bbba304e94e209c17870d6e827499 2024-11-19T05:26:27,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/a77468f27c734ae884f17e508700c75f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f 2024-11-19T05:26:27,235 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f, entries=150, sequenceid=415, filesize=12.0 K 2024-11-19T05:26:27,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/c14f46dc679141b99c6fc20c1f7b26fe as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe 2024-11-19T05:26:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe, entries=150, sequenceid=415, filesize=12.0 K 2024-11-19T05:26:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/f18bbba304e94e209c17870d6e827499 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f18bbba304e94e209c17870d6e827499 2024-11-19T05:26:27,247 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f18bbba304e94e209c17870d6e827499, entries=150, sequenceid=415, filesize=12.0 K 2024-11-19T05:26:27,248 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=127.47 KB/130530 for c8584c7fbd7ef0113dc3083d3d909de9 in 588ms, sequenceid=415, compaction requested=true 2024-11-19T05:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-19T05:26:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-19T05:26:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:27,255 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:27,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:26:27,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-19T05:26:27,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-19T05:26:27,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 769 msec 2024-11-19T05:26:27,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/85427764b0ee4ad58f8867c41b191d6a is 50, key is test_row_0/A:col10/1731993987224/Put/seqid=0 2024-11-19T05:26:27,271 INFO [Thread-1887 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1887. 2024-11-19T05:26:27,271 DEBUG [Thread-1887 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,271 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,271 DEBUG [Thread-1887 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,273 INFO [Thread-1881 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1881. 2024-11-19T05:26:27,273 DEBUG [Thread-1881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,273 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,273 DEBUG [Thread-1881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,278 INFO [Thread-1885 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1885. 2024-11-19T05:26:27,278 DEBUG [Thread-1885 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,278 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,279 DEBUG [Thread-1885 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,279 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-19T05:26:27,279 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,279 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,279 INFO [Thread-1889 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1889. 2024-11-19T05:26:27,279 DEBUG [Thread-1889 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,279 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,279 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,279 DEBUG [Thread-1889 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742328_1504 (size=14741) 2024-11-19T05:26:27,283 INFO [Thread-1883 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1883. 2024-11-19T05:26:27,283 DEBUG [Thread-1883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:27,283 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,283 DEBUG [Thread-1883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:27,283 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-19T05:26:27,647 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:27,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/85427764b0ee4ad58f8867c41b191d6a 2024-11-19T05:26:27,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/5685966787164a4abdd6cf64b0797343 is 50, key is test_row_0/B:col10/1731993987224/Put/seqid=0 2024-11-19T05:26:27,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742329_1505 (size=12301) 2024-11-19T05:26:28,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/5685966787164a4abdd6cf64b0797343 2024-11-19T05:26:28,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/95ad63ca8275405d9af0053a1aac52cd is 50, key is test_row_0/C:col10/1731993987224/Put/seqid=0 2024-11-19T05:26:28,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742330_1506 (size=12301) 2024-11-19T05:26:28,429 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-19T05:26:28,429 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:28,429 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,429 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,429 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:28,458 INFO [Thread-1878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1878. 2024-11-19T05:26:28,458 DEBUG [Thread-1878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:28,458 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,458 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:28,469 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-19T05:26:28,469 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:28,469 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,469 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,469 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:28,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/95ad63ca8275405d9af0053a1aac52cd 2024-11-19T05:26:28,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/85427764b0ee4ad58f8867c41b191d6a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a 2024-11-19T05:26:28,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a, entries=200, sequenceid=438, filesize=14.4 K 2024-11-19T05:26:28,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/5685966787164a4abdd6cf64b0797343 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343 2024-11-19T05:26:28,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343, entries=150, sequenceid=438, filesize=12.0 K 2024-11-19T05:26:28,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/95ad63ca8275405d9af0053a1aac52cd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/95ad63ca8275405d9af0053a1aac52cd 2024-11-19T05:26:28,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/95ad63ca8275405d9af0053a1aac52cd, entries=150, sequenceid=438, filesize=12.0 K 2024-11-19T05:26:28,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for c8584c7fbd7ef0113dc3083d3d909de9 in 1266ms, sequenceid=438, compaction requested=true 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:28,522 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:28,522 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c8584c7fbd7ef0113dc3083d3d909de9:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:28,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52564 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/B is initiating minor compaction (all files) 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): c8584c7fbd7ef0113dc3083d3d909de9/A is initiating minor compaction (all files) 2024-11-19T05:26:28,523 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/B in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,523 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c8584c7fbd7ef0113dc3083d3d909de9/A in TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,523 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/bc7330a238d64a8f85c0a5626cdf66d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=48.9 K 2024-11-19T05:26:28,523 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/fa2620346d4c427983c3759d20081380, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp, totalSize=51.3 K 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7330a238d64a8f85c0a5626cdf66d2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731993983890 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa2620346d4c427983c3759d20081380, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731993983890 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d73ed7142644d868c330d900403f0dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731993984012 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fef224affb44d64b17ba491e7e0b29b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731993984012 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c14f46dc679141b99c6fc20c1f7b26fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731993984802 2024-11-19T05:26:28,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77468f27c734ae884f17e508700c75f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731993984802 2024-11-19T05:26:28,524 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5685966787164a4abdd6cf64b0797343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731993987224 2024-11-19T05:26:28,524 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85427764b0ee4ad58f8867c41b191d6a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731993987196 2024-11-19T05:26:28,529 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-19T05:26:28,529 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:28,529 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,529 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:28,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:26:28,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 293 2024-11-19T05:26:28,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-19T05:26:28,529 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1351 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4049 rows 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1348 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4037 rows 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1349 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4042 rows 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1346 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4033 rows 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1347 2024-11-19T05:26:28,530 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4036 rows 2024-11-19T05:26:28,530 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:28,530 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:26:28,530 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:28,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:28,530 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/3e0393eddb1c49c8b08eee88fcd8a050 is 50, key is test_row_0/B:col10/1731993987224/Put/seqid=0 2024-11-19T05:26:28,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:26:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:28,533 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c8584c7fbd7ef0113dc3083d3d909de9#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:28,533 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0e95fb5097ce464daf0023f1d1d72121 is 50, key is test_row_0/A:col10/1731993987224/Put/seqid=0 2024-11-19T05:26:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:28,535 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993988535"}]},"ts":"1731993988535"} 2024-11-19T05:26:28,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:28,536 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:26:28,537 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:26:28,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:26:28,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, UNASSIGN}] 2024-11-19T05:26:28,539 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, UNASSIGN 2024-11-19T05:26:28,540 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c8584c7fbd7ef0113dc3083d3d909de9, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:28,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, UNASSIGN because future has completed 2024-11-19T05:26:28,541 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:26:28,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:28,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742331_1507 (size=13357) 2024-11-19T05:26:28,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742332_1508 (size=13357) 2024-11-19T05:26:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:28,693 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:28,693 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:26:28,693 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing c8584c7fbd7ef0113dc3083d3d909de9, disabling compactions & flushes 2024-11-19T05:26:28,693 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:28,950 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/3e0393eddb1c49c8b08eee88fcd8a050 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3e0393eddb1c49c8b08eee88fcd8a050 2024-11-19T05:26:28,950 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/0e95fb5097ce464daf0023f1d1d72121 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0e95fb5097ce464daf0023f1d1d72121 2024-11-19T05:26:28,953 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/B of c8584c7fbd7ef0113dc3083d3d909de9 into 3e0393eddb1c49c8b08eee88fcd8a050(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:28,953 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c8584c7fbd7ef0113dc3083d3d909de9/A of c8584c7fbd7ef0113dc3083d3d909de9 into 0e95fb5097ce464daf0023f1d1d72121(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c8584c7fbd7ef0113dc3083d3d909de9: 2024-11-19T05:26:28,953 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/B, priority=12, startTime=1731993988522; duration=0sec 2024-11-19T05:26:28,953 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., storeName=c8584c7fbd7ef0113dc3083d3d909de9/A, priority=12, startTime=1731993988522; duration=0sec 2024-11-19T05:26:28,953 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:B 2024-11-19T05:26:28,953 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:A 2024-11-19T05:26:28,953 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. after waiting 0 ms 2024-11-19T05:26:28,953 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. because compaction request was cancelled 2024-11-19T05:26:28,953 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing c8584c7fbd7ef0113dc3083d3d909de9 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-19T05:26:28,953 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c8584c7fbd7ef0113dc3083d3d909de9:C 2024-11-19T05:26:28,956 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/659121fab6994c13a83d007a3de187fb is 50, key is test_row_0/A:col10/1731993988528/Put/seqid=0 2024-11-19T05:26:28,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742333_1509 (size=12301) 2024-11-19T05:26:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:29,360 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/659121fab6994c13a83d007a3de187fb 2024-11-19T05:26:29,365 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/52b958395427420199cc6012059f089b is 50, key is test_row_0/B:col10/1731993988528/Put/seqid=0 2024-11-19T05:26:29,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742334_1510 (size=12301) 2024-11-19T05:26:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:29,769 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/52b958395427420199cc6012059f089b 2024-11-19T05:26:29,775 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e88dcf07da0e4dd9801ceaed127f2113 is 50, key is test_row_0/C:col10/1731993988528/Put/seqid=0 2024-11-19T05:26:29,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742335_1511 (size=12301) 2024-11-19T05:26:30,180 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e88dcf07da0e4dd9801ceaed127f2113 2024-11-19T05:26:30,184 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/A/659121fab6994c13a83d007a3de187fb as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/659121fab6994c13a83d007a3de187fb 2024-11-19T05:26:30,187 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/659121fab6994c13a83d007a3de187fb, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T05:26:30,188 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/B/52b958395427420199cc6012059f089b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/52b958395427420199cc6012059f089b 2024-11-19T05:26:30,191 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/52b958395427420199cc6012059f089b, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T05:26:30,192 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/.tmp/C/e88dcf07da0e4dd9801ceaed127f2113 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e88dcf07da0e4dd9801ceaed127f2113 2024-11-19T05:26:30,195 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e88dcf07da0e4dd9801ceaed127f2113, entries=150, sequenceid=449, filesize=12.0 K 2024-11-19T05:26:30,196 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for c8584c7fbd7ef0113dc3083d3d909de9 in 1243ms, sequenceid=449, compaction requested=true 2024-11-19T05:26:30,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/b7ccac96e7b5484aa37c2e539ab23d87, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9de59f6272d645cfafde246b2984fb5e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/896e0d4bbc1f4f77afdda5765e4e599c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e8c047278c1f49f688819615151bd53b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a183d59b3fba4aaa8b589a311a724021, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7d6c947430154569bb5c0402d6f5b1e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6f0de4ba0f374f47936d557ae70b7faa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/fa2620346d4c427983c3759d20081380, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a] to archive 2024-11-19T05:26:30,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:30,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7b1985637603458892d72ec6f4b141a7 2024-11-19T05:26:30,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/accc060df67d4bf89b000893eef8e30b 2024-11-19T05:26:30,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/b7ccac96e7b5484aa37c2e539ab23d87 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/b7ccac96e7b5484aa37c2e539ab23d87 2024-11-19T05:26:30,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/755f4723d6434b66ab3aa170966946c7 2024-11-19T05:26:30,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/168e569e027343ec9f78152336a1c4dd 2024-11-19T05:26:30,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/3478f0c079f241aeb9f8b03f5e582db2 2024-11-19T05:26:30,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6c05582cff714b738c2475c09fe8736a 2024-11-19T05:26:30,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9de59f6272d645cfafde246b2984fb5e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9de59f6272d645cfafde246b2984fb5e 2024-11-19T05:26:30,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c4db25aad2264c5f82ee2c4903d113e9 2024-11-19T05:26:30,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/84322771f600497dbf00367ba328fe37 2024-11-19T05:26:30,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/896e0d4bbc1f4f77afdda5765e4e599c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/896e0d4bbc1f4f77afdda5765e4e599c 2024-11-19T05:26:30,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/204114e1d0354da9acd292b83ddb0b8b 2024-11-19T05:26:30,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/333cba1b7c3f43b5ac93e80bf8a1434e 2024-11-19T05:26:30,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e77d38edc0cb4cd49b93cc7716166e89 2024-11-19T05:26:30,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e8c047278c1f49f688819615151bd53b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/e8c047278c1f49f688819615151bd53b 2024-11-19T05:26:30,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0062aaa511db4d3cbdfa14e3cac43b9a 2024-11-19T05:26:30,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/c619c695e3a045d29dfe55825001bedd 2024-11-19T05:26:30,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a183d59b3fba4aaa8b589a311a724021 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a183d59b3fba4aaa8b589a311a724021 2024-11-19T05:26:30,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/574ef46607cf439d90766f274fd99046 2024-11-19T05:26:30,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0a88b20261a946398751d80990420f4a 2024-11-19T05:26:30,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7d6c947430154569bb5c0402d6f5b1e0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7d6c947430154569bb5c0402d6f5b1e0 2024-11-19T05:26:30,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/9ca51d768eff40fa8ca272705f91433d 2024-11-19T05:26:30,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/f5a2cb8eae4a4837a32f79b62eda601a 2024-11-19T05:26:30,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6f0de4ba0f374f47936d557ae70b7faa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/6f0de4ba0f374f47936d557ae70b7faa 2024-11-19T05:26:30,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/7993efba5b4c4e32823677ba899d2c2c 2024-11-19T05:26:30,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/4d3b2f424d304ef59233ae23a0048ba7 2024-11-19T05:26:30,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/90528aa734b541b8baefb7a06d8a55b4 2024-11-19T05:26:30,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/fa2620346d4c427983c3759d20081380 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/fa2620346d4c427983c3759d20081380 2024-11-19T05:26:30,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/8fef224affb44d64b17ba491e7e0b29b 2024-11-19T05:26:30,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/a77468f27c734ae884f17e508700c75f 2024-11-19T05:26:30,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/85427764b0ee4ad58f8867c41b191d6a 2024-11-19T05:26:30,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/55933f718a55411a9f6b28592218a917, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/96b92f467de4480ba3e075f03af53791, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6e1352dd86cb459f815bdd1ae5aa009b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d37742f2601443adb07366f7a3c9a4b9, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/29028fbe4f3840029331f08fad764c38, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ba63371193bf4804ac8f35b29b0802ab, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/31f7821621984862a9c3f2165ecbbc77, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/bc7330a238d64a8f85c0a5626cdf66d2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343] to archive 2024-11-19T05:26:30,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:30,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/010292020cff4230825df24f85044af3 2024-11-19T05:26:30,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ceeb7ad5bbd34da9a7c41a8b8fb3d7a7 2024-11-19T05:26:30,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/55933f718a55411a9f6b28592218a917 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/55933f718a55411a9f6b28592218a917 2024-11-19T05:26:30,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/1e7e85e452ba42c396218ad6ffadd7bb 2024-11-19T05:26:30,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3300cf9cd6564f8fb99f7194365277f9 2024-11-19T05:26:30,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/07bcbbd9f4e940e2b64b5aba57301641 2024-11-19T05:26:30,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/96b92f467de4480ba3e075f03af53791 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/96b92f467de4480ba3e075f03af53791 2024-11-19T05:26:30,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/df537d9d41934b81a077b071714cc4a3 2024-11-19T05:26:30,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d0ae54bdc65c499f92420d67e8550563 2024-11-19T05:26:30,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6e1352dd86cb459f815bdd1ae5aa009b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6e1352dd86cb459f815bdd1ae5aa009b 2024-11-19T05:26:30,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/b1c6d297ef0846719d30524837c5e727 2024-11-19T05:26:30,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e3c2e0d127b54312a6729b56a7e02fce 2024-11-19T05:26:30,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/39fd872cc760484181c269beb0bff3f8 2024-11-19T05:26:30,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d37742f2601443adb07366f7a3c9a4b9 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/d37742f2601443adb07366f7a3c9a4b9 2024-11-19T05:26:30,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/6bcd9cb520bd413aa0d886dd8807c213 2024-11-19T05:26:30,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e35c93bd8c6a433aaa904b03461b7b02 2024-11-19T05:26:30,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/29028fbe4f3840029331f08fad764c38 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/29028fbe4f3840029331f08fad764c38 2024-11-19T05:26:30,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e89cfd55ce8147bf82d5073ea2f2f1c7 2024-11-19T05:26:30,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/aeaff6d456f540109b3e2473a2245efd 2024-11-19T05:26:30,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ba63371193bf4804ac8f35b29b0802ab to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/ba63371193bf4804ac8f35b29b0802ab 2024-11-19T05:26:30,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/033e9095ad8745e6a082f93cd4c82073 2024-11-19T05:26:30,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/2ccd6d2de1904d03acb77cdf908d2c26 2024-11-19T05:26:30,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/31f7821621984862a9c3f2165ecbbc77 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/31f7821621984862a9c3f2165ecbbc77 2024-11-19T05:26:30,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/34a4c621a23a46f78b6d33cffc6cc1e1 2024-11-19T05:26:30,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/e938df5d42b54c4389bfd08a6d810f88 2024-11-19T05:26:30,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/329514a941b94191b9f6deba622368e0 2024-11-19T05:26:30,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/bc7330a238d64a8f85c0a5626cdf66d2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/bc7330a238d64a8f85c0a5626cdf66d2 2024-11-19T05:26:30,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/82158480b4ed439981809150b8f6127e 2024-11-19T05:26:30,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/4d73ed7142644d868c330d900403f0dd 2024-11-19T05:26:30,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/c14f46dc679141b99c6fc20c1f7b26fe 2024-11-19T05:26:30,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/5685966787164a4abdd6cf64b0797343 2024-11-19T05:26:30,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/8bcaaec9f3774dc7a928f012209a999d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5f4c86f3607a4456b055456a2b055dc3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36a3531fe15e4ac6844ea31e410fceac, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/9e054f9be706461ea8a76455901bd0f5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/35ffb965301943d991d708bbb9971206, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0fe99a8ebf07481a8af25b0c627003e7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36087400bf06452fb88d3b788d07baea, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06] to archive 2024-11-19T05:26:30,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:30,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/ed25998e3e09495687fbbf180511d6a0 2024-11-19T05:26:30,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/eda225a9781e438fa3bb4d81092d0ba2 2024-11-19T05:26:30,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/8bcaaec9f3774dc7a928f012209a999d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/8bcaaec9f3774dc7a928f012209a999d 2024-11-19T05:26:30,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/acf76ec75b9d4dc18d01120a102c03fd 2024-11-19T05:26:30,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/72b24603143642a3bf787812669c97b7 2024-11-19T05:26:30,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5e0cf2a6e3b644bdbb090a56d80ba337 2024-11-19T05:26:30,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5f4c86f3607a4456b055456a2b055dc3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5f4c86f3607a4456b055456a2b055dc3 2024-11-19T05:26:30,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/1433881bfa6f4dad8d312e89f35867ae 2024-11-19T05:26:30,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/58f059dae50440bd9296af43c0d5e78a 2024-11-19T05:26:30,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36a3531fe15e4ac6844ea31e410fceac to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36a3531fe15e4ac6844ea31e410fceac 2024-11-19T05:26:30,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/803ddb8032dd44729ec3004b3ed667af 2024-11-19T05:26:30,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e9193a8004d9469e9806fa2e5f0a3471 2024-11-19T05:26:30,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/d19a7d577b4a4ace8271c458175deb09 2024-11-19T05:26:30,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/9e054f9be706461ea8a76455901bd0f5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/9e054f9be706461ea8a76455901bd0f5 2024-11-19T05:26:30,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/2303ba352af54510ae0e74790ec31f56 2024-11-19T05:26:30,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/5aedaccff57640bea147d4d5d4399ec4 2024-11-19T05:26:30,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/35ffb965301943d991d708bbb9971206 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/35ffb965301943d991d708bbb9971206 2024-11-19T05:26:30,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36ce3c04d8cc44b89cbc72c4cb49800f 2024-11-19T05:26:30,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6174006de5d14c4b9fdf096780ecd8b3 2024-11-19T05:26:30,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0fe99a8ebf07481a8af25b0c627003e7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0fe99a8ebf07481a8af25b0c627003e7 2024-11-19T05:26:30,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0cb95efeec02441fa76a27b899ea7c1e 2024-11-19T05:26:30,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/6df6a7b9cecd40a7983fcb8f66e10b81 2024-11-19T05:26:30,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36087400bf06452fb88d3b788d07baea to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/36087400bf06452fb88d3b788d07baea 2024-11-19T05:26:30,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/239b47546e7144db917f0b06b606068d 2024-11-19T05:26:30,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/cb2107f4c7734838b9379e644b7e814b 2024-11-19T05:26:30,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f751963210534a7896c258cba83fa6aa 2024-11-19T05:26:30,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/87f5d58abaf44e78a9bd2b256ee70c06 2024-11-19T05:26:30,279 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/recovered.edits/452.seqid, newMaxSeqId=452, maxSeqId=1 2024-11-19T05:26:30,279 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. 2024-11-19T05:26:30,279 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for c8584c7fbd7ef0113dc3083d3d909de9: Waiting for close lock at 1731993988693Running coprocessor pre-close hooks at 1731993988693Disabling compacts and flushes for region at 1731993988693Disabling writes for close at 1731993988953 (+260 ms)Obtaining lock to block concurrent updates at 1731993988953Preparing flush snapshotting stores in c8584c7fbd7ef0113dc3083d3d909de9 at 1731993988953Finished memstore snapshotting TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9., syncing WAL and waiting on mvcc, flushsize=dataSize=41220, getHeapSize=108720, getOffHeapSize=0, getCellsCount=900 at 1731993988953Flushing stores of TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9. at 1731993988954 (+1 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/A: creating writer at 1731993988954Flushing c8584c7fbd7ef0113dc3083d3d909de9/A: appending metadata at 1731993988956 (+2 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/A: closing flushed file at 1731993988956Flushing c8584c7fbd7ef0113dc3083d3d909de9/B: creating writer at 1731993989363 (+407 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/B: appending metadata at 1731993989365 (+2 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/B: closing flushed file at 1731993989365Flushing c8584c7fbd7ef0113dc3083d3d909de9/C: creating writer at 1731993989772 (+407 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/C: appending metadata at 1731993989775 (+3 ms)Flushing c8584c7fbd7ef0113dc3083d3d909de9/C: closing flushed file at 1731993989775Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@396cefa7: reopening flushed file at 1731993990183 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dbc8db0: reopening flushed file at 1731993990187 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25658813: reopening flushed file at 1731993990191 (+4 ms)Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for c8584c7fbd7ef0113dc3083d3d909de9 in 1243ms, sequenceid=449, compaction requested=true at 1731993990196 (+5 ms)Writing region close event to WAL at 1731993990276 (+80 ms)Running coprocessor post-close hooks at 1731993990279 (+3 ms)Closed at 1731993990279 2024-11-19T05:26:30,281 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:30,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=c8584c7fbd7ef0113dc3083d3d909de9, regionState=CLOSED 2024-11-19T05:26:30,283 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:30,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-19T05:26:30,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure c8584c7fbd7ef0113dc3083d3d909de9, server=f990f635ffd0,38449,1731993838792 in 1.7430 sec 2024-11-19T05:26:30,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-19T05:26:30,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c8584c7fbd7ef0113dc3083d3d909de9, UNASSIGN in 1.7470 sec 2024-11-19T05:26:30,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-19T05:26:30,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7500 sec 2024-11-19T05:26:30,290 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993990290"}]},"ts":"1731993990290"} 2024-11-19T05:26:30,291 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:26:30,291 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:26:30,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7610 sec 2024-11-19T05:26:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-19T05:26:30,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:30,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:26:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,679 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T05:26:30,680 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,681 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:30,683 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/recovered.edits] 2024-11-19T05:26:30,685 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0e95fb5097ce464daf0023f1d1d72121 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/0e95fb5097ce464daf0023f1d1d72121 2024-11-19T05:26:30,686 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/659121fab6994c13a83d007a3de187fb to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/A/659121fab6994c13a83d007a3de187fb 2024-11-19T05:26:30,688 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3e0393eddb1c49c8b08eee88fcd8a050 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/3e0393eddb1c49c8b08eee88fcd8a050 2024-11-19T05:26:30,689 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/52b958395427420199cc6012059f089b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/B/52b958395427420199cc6012059f089b 2024-11-19T05:26:30,690 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0c857d4ca46643a48667ad6be352c057 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/0c857d4ca46643a48667ad6be352c057 2024-11-19T05:26:30,691 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/95ad63ca8275405d9af0053a1aac52cd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/95ad63ca8275405d9af0053a1aac52cd 2024-11-19T05:26:30,692 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/a18140c7f8fb49e3a8069edc8bf674ae to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/a18140c7f8fb49e3a8069edc8bf674ae 2024-11-19T05:26:30,693 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e88dcf07da0e4dd9801ceaed127f2113 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/e88dcf07da0e4dd9801ceaed127f2113 2024-11-19T05:26:30,694 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f18bbba304e94e209c17870d6e827499 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/C/f18bbba304e94e209c17870d6e827499 2024-11-19T05:26:30,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/recovered.edits/452.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9/recovered.edits/452.seqid 2024-11-19T05:26:30,696 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/c8584c7fbd7ef0113dc3083d3d909de9 2024-11-19T05:26:30,696 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:26:30,698 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,700 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:26:30,702 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:26:30,703 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,703 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:26:30,703 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731993990703"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:30,705 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:26:30,705 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c8584c7fbd7ef0113dc3083d3d909de9, NAME => 'TestAcidGuarantees,,1731993965060.c8584c7fbd7ef0113dc3083d3d909de9.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:26:30,705 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:26:30,705 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731993990705"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:30,707 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:26:30,707 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,708 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-19T05:26:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-19T05:26:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:26:30,790 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:30,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 242), OpenFileDescriptor=449 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=554 (was 534) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=11640 (was 11805) 2024-11-19T05:26:30,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=554, ProcessCount=11, AvailableMemoryMB=11640 2024-11-19T05:26:30,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:26:30,805 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:26:30,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:30,807 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T05:26:30,807 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:30,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-19T05:26:30,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:30,808 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T05:26:30,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742336_1512 (size=959) 2024-11-19T05:26:30,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:31,214 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad 2024-11-19T05:26:31,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742337_1513 (size=53) 2024-11-19T05:26:31,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3086702af0f4dab24249af64f43614f1, disabling compactions & flushes 2024-11-19T05:26:31,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. after waiting 0 ms 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,623 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,623 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3086702af0f4dab24249af64f43614f1: Waiting for close lock at 1731993991623Disabling compacts and flushes for region at 1731993991623Disabling writes for close at 1731993991623Writing region close event to WAL at 1731993991623Closed at 1731993991623 2024-11-19T05:26:31,624 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T05:26:31,624 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731993991624"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731993991624"}]},"ts":"1731993991624"} 2024-11-19T05:26:31,625 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T05:26:31,626 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T05:26:31,626 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993991626"}]},"ts":"1731993991626"} 2024-11-19T05:26:31,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-19T05:26:31,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, ASSIGN}] 2024-11-19T05:26:31,629 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, ASSIGN 2024-11-19T05:26:31,629 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, ASSIGN; state=OFFLINE, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=false 2024-11-19T05:26:31,780 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:31,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, ASSIGN because future has completed 2024-11-19T05:26:31,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:31,935 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,935 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:26:31,935 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,935 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:26:31,936 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,936 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,937 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,938 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName A 2024-11-19T05:26:31,938 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:31,938 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:31,938 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,939 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName B 2024-11-19T05:26:31,939 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:31,939 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:31,939 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,940 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName C 2024-11-19T05:26:31,940 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:31,941 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:31,941 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,941 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,942 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,942 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,942 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,943 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:26:31,944 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,945 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T05:26:31,946 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened 3086702af0f4dab24249af64f43614f1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61113706, jitterRate=-0.08933481574058533}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:26:31,946 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:31,947 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for 3086702af0f4dab24249af64f43614f1: Running coprocessor pre-open hook at 1731993991936Writing region info on filesystem at 1731993991936Initializing all the Stores at 1731993991936Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993991936Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993991936Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993991936Cleaning up temporary data from old regions at 1731993991942 (+6 ms)Running coprocessor post-open hooks at 1731993991946 (+4 ms)Region opened successfully at 1731993991946 2024-11-19T05:26:31,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:31,947 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., pid=138, masterSystemTime=1731993991933 2024-11-19T05:26:31,949 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,949 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:31,949 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=OPEN, openSeqNum=2, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:31,950 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34055 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=f990f635ffd0,38449,1731993838792, table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T05:26:31,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:31,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-19T05:26:31,953 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 in 169 msec 2024-11-19T05:26:31,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-19T05:26:31,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, ASSIGN in 325 msec 2024-11-19T05:26:31,955 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T05:26:31,955 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731993991955"}]},"ts":"1731993991955"} 2024-11-19T05:26:31,956 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-19T05:26:31,957 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T05:26:31,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-19T05:26:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-19T05:26:32,957 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:32,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:32,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:32,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:32,959 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab57b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:32,959 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:32,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:32,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:32,961 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:32,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c8e17df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:32,962 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:32,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:32,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:32,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:32,964 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T05:26:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is f990f635ffd0,34055,1731993838016 2024-11-19T05:26:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d56ef31 2024-11-19T05:26:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T05:26:32,966 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T05:26:32,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-19T05:26:32,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T05:26:32,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:32,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742338_1514 (size=995) 2024-11-19T05:26:33,378 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-19T05:26:33,378 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-19T05:26:33,381 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:26:33,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE}] 2024-11-19T05:26:33,383 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE 2024-11-19T05:26:33,384 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE because future has completed 2024-11-19T05:26:33,385 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:26:33,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:33,537 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,537 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:26:33,537 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 3086702af0f4dab24249af64f43614f1, disabling compactions & flushes 2024-11-19T05:26:33,537 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,537 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,537 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. after waiting 0 ms 2024-11-19T05:26:33,537 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,540 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-19T05:26:33,541 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,541 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 3086702af0f4dab24249af64f43614f1: Waiting for close lock at 1731993993537Running coprocessor pre-close hooks at 1731993993537Disabling compacts and flushes for region at 1731993993537Disabling writes for close at 1731993993537Writing region close event to WAL at 1731993993538 (+1 ms)Running coprocessor post-close hooks at 1731993993541 (+3 ms)Closed at 1731993993541 2024-11-19T05:26:33,541 WARN [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: 3086702af0f4dab24249af64f43614f1 to self. 2024-11-19T05:26:33,543 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,543 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=CLOSED 2024-11-19T05:26:33,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:33,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-19T05:26:33,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 in 160 msec 2024-11-19T05:26:33,546 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE; state=CLOSED, location=f990f635ffd0,38449,1731993838792; forceNewPlan=false, retain=true 2024-11-19T05:26:33,697 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=OPENING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,698 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE because future has completed 2024-11-19T05:26:33,698 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:33,853 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T05:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T05:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,854 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,855 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName A 2024-11-19T05:26:33,856 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:33,856 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:33,856 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,857 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName B 2024-11-19T05:26:33,857 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:33,857 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:33,857 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,858 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3086702af0f4dab24249af64f43614f1 columnFamilyName C 2024-11-19T05:26:33,858 DEBUG [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:33,858 INFO [StoreOpener-3086702af0f4dab24249af64f43614f1-1 {}] regionserver.HStore(327): Store=3086702af0f4dab24249af64f43614f1/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T05:26:33,858 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,859 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,860 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,860 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,860 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,861 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-19T05:26:33,862 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,862 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened 3086702af0f4dab24249af64f43614f1; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67437213, jitterRate=0.004892781376838684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-19T05:26:33,862 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,863 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for 3086702af0f4dab24249af64f43614f1: Running coprocessor pre-open hook at 1731993993853Writing region info on filesystem at 1731993993853Initializing all the Stores at 1731993993854 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993993854Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993993854Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731993993854Cleaning up temporary data from old regions at 1731993993860 (+6 ms)Running coprocessor post-open hooks at 1731993993862 (+2 ms)Region opened successfully at 1731993993863 (+1 ms) 2024-11-19T05:26:33,863 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., pid=143, masterSystemTime=1731993993850 2024-11-19T05:26:33,865 DEBUG [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,865 INFO [RS_OPEN_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:33,865 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=OPEN, openSeqNum=5, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:33,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-19T05:26:33,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 in 169 msec 2024-11-19T05:26:33,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-19T05:26:33,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, REOPEN/MOVE in 486 msec 2024-11-19T05:26:33,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-19T05:26:33,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-11-19T05:26:33,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-11-19T05:26:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-19T05:26:33,873 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:33,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,875 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673ede3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,876 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75c9ffa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462f0ad4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f81f446, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f553b81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,882 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183b4f82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,884 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acad5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7f264c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,887 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f58a10b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,888 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c43435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,890 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234a1e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,891 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,891 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e329896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,892 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,892 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331c43e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,893 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,893 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,894 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5eef2412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,895 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9ef6af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,896 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,896 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4887727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,899 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,899 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52606c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,901 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57344c83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,902 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f990f635ffd0,34055,-1 for getting cluster id 2024-11-19T05:26:33,902 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T05:26:33,902 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ec00e422-e444-4d95-8b94-b69e99cfec7d' 2024-11-19T05:26:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T05:26:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ec00e422-e444-4d95-8b94-b69e99cfec7d" 2024-11-19T05:26:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457b0964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f990f635ffd0,34055,-1] 2024-11-19T05:26:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T05:26:33,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:33,903 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T05:26:33,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb2d8b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T05:26:33,904 DEBUG [Thread-2315 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,904 DEBUG [Thread-2317 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,905 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,905 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,905 DEBUG [Thread-2319 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,905 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,906 DEBUG [Thread-2319 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,906 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,906 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,906 DEBUG [Thread-2321 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,906 DEBUG [Thread-2319 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,907 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-19T05:26:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,909 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:33,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:33,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,911 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,912 DEBUG [Thread-2313 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,913 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T05:26:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,914 DEBUG [Thread-2313 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f990f635ffd0,38449,1731993838792, seqNum=-1] 2024-11-19T05:26:33,914 DEBUG [Thread-2313 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T05:26:33,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T05:26:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5] 2024-11-19T05:26:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,917 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:33,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:33,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194c34cd8680814c93810a003f6ad1759e_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993993916/Put/seqid=0 2024-11-19T05:26:33,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994053942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994053942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994053946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994053947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994053947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742339_1515 (size=12154) 2024-11-19T05:26:33,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:33,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194c34cd8680814c93810a003f6ad1759e_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c34cd8680814c93810a003f6ad1759e_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:33,975 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/19dbeb9785284297a418859430ed8c7b, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:33,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/19dbeb9785284297a418859430ed8c7b is 175, key is test_row_0/A:col10/1731993993916/Put/seqid=0 2024-11-19T05:26:33,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742340_1516 (size=30955) 2024-11-19T05:26:33,985 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/19dbeb9785284297a418859430ed8c7b 2024-11-19T05:26:34,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/5f6a2188299744cf938208c0bd9af48d is 50, key is test_row_0/B:col10/1731993993916/Put/seqid=0 2024-11-19T05:26:34,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742341_1517 (size=12001) 2024-11-19T05:26:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:34,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994054057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994054057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994054057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994054057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994054057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994054267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,271 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/5f6a2188299744cf938208c0bd9af48d 2024-11-19T05:26:34,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ede76d269d12462eaeeb2e814af19d68 is 50, key is test_row_0/C:col10/1731993993916/Put/seqid=0 2024-11-19T05:26:34,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742342_1518 (size=12001) 2024-11-19T05:26:34,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994054577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994054577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994054577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994054577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994054577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:34,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:34,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:34,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ede76d269d12462eaeeb2e814af19d68 2024-11-19T05:26:34,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/19dbeb9785284297a418859430ed8c7b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b 2024-11-19T05:26:34,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-19T05:26:34,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/5f6a2188299744cf938208c0bd9af48d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d 2024-11-19T05:26:34,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:26:34,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ede76d269d12462eaeeb2e814af19d68 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68 2024-11-19T05:26:34,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68, entries=150, sequenceid=16, filesize=11.7 K 2024-11-19T05:26:34,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 3086702af0f4dab24249af64f43614f1 in 938ms, sequenceid=16, compaction requested=false 2024-11-19T05:26:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-19T05:26:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:34,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-19T05:26:34,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:34,980 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c9a72d7804f44f5ab564cf9de2ba626_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993993945/Put/seqid=0 2024-11-19T05:26:35,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742343_1519 (size=12154) 2024-11-19T05:26:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:35,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994055092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994055092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,097 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994055095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994055096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994055097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,100 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994055207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994055207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994055207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994055207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994055207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994055417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994055418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994055418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994055424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994055424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:35,429 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411196c9a72d7804f44f5ab564cf9de2ba626_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c9a72d7804f44f5ab564cf9de2ba626_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:35,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a6aa94388a494bc59d7ea7a2e4a541bd, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:35,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a6aa94388a494bc59d7ea7a2e4a541bd is 175, key is test_row_0/A:col10/1731993993945/Put/seqid=0 2024-11-19T05:26:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742344_1520 (size=30955) 2024-11-19T05:26:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994055727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994055727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994055727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994055727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994055728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:35,796 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T05:26:35,846 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a6aa94388a494bc59d7ea7a2e4a541bd 2024-11-19T05:26:35,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c is 50, key is test_row_0/B:col10/1731993993945/Put/seqid=0 2024-11-19T05:26:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742345_1521 (size=12001) 2024-11-19T05:26:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:36,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994056237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994056237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994056237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994056237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994056237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:36,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:36,256 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c 2024-11-19T05:26:36,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/13d6e25fe2b842dcbc44eba1b03deae3 is 50, key is test_row_0/C:col10/1731993993945/Put/seqid=0 2024-11-19T05:26:36,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742346_1522 (size=12001) 2024-11-19T05:26:36,264 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/13d6e25fe2b842dcbc44eba1b03deae3 2024-11-19T05:26:36,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a6aa94388a494bc59d7ea7a2e4a541bd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd 2024-11-19T05:26:36,269 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd, entries=150, sequenceid=41, filesize=30.2 K 2024-11-19T05:26:36,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c 2024-11-19T05:26:36,278 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:26:36,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/13d6e25fe2b842dcbc44eba1b03deae3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3 2024-11-19T05:26:36,282 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3, entries=150, sequenceid=41, filesize=11.7 K 2024-11-19T05:26:36,283 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3086702af0f4dab24249af64f43614f1 in 1303ms, sequenceid=41, compaction requested=false 2024-11-19T05:26:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-19T05:26:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-19T05:26:36,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-19T05:26:36,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3750 sec 2024-11-19T05:26:36,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.3800 sec 2024-11-19T05:26:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:37,250 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:37,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-19T05:26:37,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192feed167265a432480a498d94bfd177b_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:37,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742347_1523 (size=17034) 2024-11-19T05:26:37,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:37,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411192feed167265a432480a498d94bfd177b_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192feed167265a432480a498d94bfd177b_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:37,263 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1049af3858814db5a9dc89ab2a51f86c, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:37,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1049af3858814db5a9dc89ab2a51f86c is 175, key is test_row_0/A:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:37,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742348_1524 (size=48139) 2024-11-19T05:26:37,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994057280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994057280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994057281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994057281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,287 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994057282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994057607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994057607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994057607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994057607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994057607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1049af3858814db5a9dc89ab2a51f86c 2024-11-19T05:26:37,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/1acf6bef3d97489e9ff8fede7bc4bbc4 is 50, key is test_row_0/B:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:37,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742349_1525 (size=12001) 2024-11-19T05:26:37,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/1acf6bef3d97489e9ff8fede7bc4bbc4 2024-11-19T05:26:37,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/4cf613c883784147a63f9ab530aacae6 is 50, key is test_row_0/C:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:37,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742350_1526 (size=12001) 2024-11-19T05:26:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994057917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994057917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994057917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994057917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:37,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994057917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:37,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-19T05:26:38,078 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:38,078 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994058427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994058427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994058427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994058427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:38,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994058427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:38,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:38,487 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_1217569536_22 at /127.0.0.1:49176 [Receiving block BP-23751114-172.17.0.2-1731993833712:blk_1073741830_1006] {}] datanode.BlockReceiver(464): Slow flushOrSync took 408ms (threshold=300ms), isSync:false, flushTotalNanos=407846882ns, volume=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data2/, blockId=1073741830, seqno=1767 2024-11-19T05:26:38,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/4cf613c883784147a63f9ab530aacae6 2024-11-19T05:26:38,487 INFO [AsyncFSWAL-0-hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData-prefix:f990f635ffd0,34055,1731993838016 {}] wal.AbstractFSWAL(1368): Slow sync cost: 408 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33169,DS-2c89d28b-24fb-4551-a99d-6fed4db29f11,DISK]] 2024-11-19T05:26:38,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-19T05:26:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T05:26:38,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:38,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:38,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:38,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1049af3858814db5a9dc89ab2a51f86c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c 2024-11-19T05:26:38,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c, entries=250, sequenceid=54, filesize=47.0 K 2024-11-19T05:26:38,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/1acf6bef3d97489e9ff8fede7bc4bbc4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4 2024-11-19T05:26:38,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T05:26:38,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/4cf613c883784147a63f9ab530aacae6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6 2024-11-19T05:26:38,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6, entries=150, sequenceid=54, filesize=11.7 K 2024-11-19T05:26:38,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3086702af0f4dab24249af64f43614f1 in 1253ms, sequenceid=54, compaction requested=true 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:38,503 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:38,503 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:38,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:38,504 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:38,504 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:38,504 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:38,504 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:38,504 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:38,504 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:38,504 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=107.5 K 2024-11-19T05:26:38,504 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.2 K 2024-11-19T05:26:38,504 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c] 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6a2188299744cf938208c0bd9af48d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993993913 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19dbeb9785284297a418859430ed8c7b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993993913 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting c3ecd2cc94e446b4b9ecd3d1b9a8656c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993993940 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6aa94388a494bc59d7ea7a2e4a541bd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993993940 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 1acf6bef3d97489e9ff8fede7bc4bbc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:38,505 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1049af3858814db5a9dc89ab2a51f86c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:38,510 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:38,512 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:38,513 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/f295b2eae5fb42859f1e5094aa1aff5d is 50, key is test_row_0/B:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:38,515 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411191e47946fa86449a29fc280a366637eed_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:38,517 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411191e47946fa86449a29fc280a366637eed_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:38,517 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191e47946fa86449a29fc280a366637eed_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:38,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742352_1528 (size=4469) 2024-11-19T05:26:38,523 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#449 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:38,523 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a9f3d933d2834057864c0f6cbaab1a7b is 175, key is test_row_0/A:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:38,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742351_1527 (size=12104) 2024-11-19T05:26:38,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742353_1529 (size=31058) 2024-11-19T05:26:38,537 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/f295b2eae5fb42859f1e5094aa1aff5d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/f295b2eae5fb42859f1e5094aa1aff5d 2024-11-19T05:26:38,542 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into f295b2eae5fb42859f1e5094aa1aff5d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:38,542 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:38,542 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731993998503; duration=0sec 2024-11-19T05:26:38,542 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:38,542 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:38,542 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:38,543 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:38,543 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:38,543 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:38,543 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.2 K 2024-11-19T05:26:38,543 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ede76d269d12462eaeeb2e814af19d68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731993993913 2024-11-19T05:26:38,543 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 13d6e25fe2b842dcbc44eba1b03deae3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731993993940 2024-11-19T05:26:38,544 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf613c883784147a63f9ab530aacae6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:38,549 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#451 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:38,549 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7d158bfa66b74d7884f2f8c313d2a85a is 50, key is test_row_0/C:col10/1731993997250/Put/seqid=0 2024-11-19T05:26:38,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742354_1530 (size=12104) 2024-11-19T05:26:38,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T05:26:38,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-19T05:26:38,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:38,641 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:26:38,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195bee869936754a13b0eb7bb798206edd_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993997281/Put/seqid=0 2024-11-19T05:26:38,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742355_1531 (size=12154) 2024-11-19T05:26:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T05:26:38,933 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/a9f3d933d2834057864c0f6cbaab1a7b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b 2024-11-19T05:26:38,936 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into a9f3d933d2834057864c0f6cbaab1a7b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:38,936 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:38,936 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731993998503; duration=0sec 2024-11-19T05:26:38,936 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:38,936 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:38,958 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7d158bfa66b74d7884f2f8c313d2a85a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7d158bfa66b74d7884f2f8c313d2a85a 2024-11-19T05:26:38,961 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 7d158bfa66b74d7884f2f8c313d2a85a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:38,961 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:38,961 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731993998503; duration=0sec 2024-11-19T05:26:38,961 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:38,961 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:39,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:39,057 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195bee869936754a13b0eb7bb798206edd_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195bee869936754a13b0eb7bb798206edd_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f6e045d8015941a6a99de4d2977027ef, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:39,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f6e045d8015941a6a99de4d2977027ef is 175, key is test_row_0/A:col10/1731993997281/Put/seqid=0 2024-11-19T05:26:39,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742356_1532 (size=30955) 2024-11-19T05:26:39,062 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f6e045d8015941a6a99de4d2977027ef 2024-11-19T05:26:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/0246218150cb455fb7ba0cfbcd94d8d1 is 50, key is test_row_0/B:col10/1731993997281/Put/seqid=0 2024-11-19T05:26:39,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742357_1533 (size=12001) 2024-11-19T05:26:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T05:26:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:39,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:39,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994059444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994059446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994059446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994059447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994059447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,451 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,480 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/0246218150cb455fb7ba0cfbcd94d8d1 2024-11-19T05:26:39,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/75b516cf07664ac5b1c635109a1ee238 is 50, key is test_row_0/C:col10/1731993997281/Put/seqid=0 2024-11-19T05:26:39,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742358_1534 (size=12001) 2024-11-19T05:26:39,495 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/75b516cf07664ac5b1c635109a1ee238 2024-11-19T05:26:39,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f6e045d8015941a6a99de4d2977027ef as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef 2024-11-19T05:26:39,501 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef, entries=150, sequenceid=78, filesize=30.2 K 2024-11-19T05:26:39,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/0246218150cb455fb7ba0cfbcd94d8d1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1 2024-11-19T05:26:39,504 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T05:26:39,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/75b516cf07664ac5b1c635109a1ee238 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238 2024-11-19T05:26:39,507 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238, entries=150, sequenceid=78, filesize=11.7 K 2024-11-19T05:26:39,508 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 3086702af0f4dab24249af64f43614f1 in 867ms, sequenceid=78, compaction requested=false 2024-11-19T05:26:39,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:39,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:39,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-19T05:26:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-19T05:26:39,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-19T05:26:39,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0200 sec 2024-11-19T05:26:39,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4330 sec 2024-11-19T05:26:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:39,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:39,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-19T05:26:39,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119238686d614564325a6c0e3844784b510_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742359_1535 (size=12154) 2024-11-19T05:26:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994059579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994059588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994059588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994059588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-19T05:26:39,627 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:39,628 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-19T05:26:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:39,632 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:39,633 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:39,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:39,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994059697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994059697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994059697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994059697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:39,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:39,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994059907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994059907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994059907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994059907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:39,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:39,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:39,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:39,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:39,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:39,983 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119238686d614564325a6c0e3844784b510_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238686d614564325a6c0e3844784b510_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:39,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/8674b72a08b246ccb488d7dddad41c4f, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:39,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/8674b72a08b246ccb488d7dddad41c4f is 175, key is test_row_0/A:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:39,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742360_1536 (size=30955) 2024-11-19T05:26:40,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994060217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994060218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994060218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994060218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:40,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/8674b72a08b246ccb488d7dddad41c4f 2024-11-19T05:26:40,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/4ce7df5b94e644a2a9545382cd63367e is 50, key is test_row_0/B:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:40,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742361_1537 (size=12001) 2024-11-19T05:26:40,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994060727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994060728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994060728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994060728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:40,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/4ce7df5b94e644a2a9545382cd63367e 2024-11-19T05:26:40,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ef72464a19c48de9adafc263d066afa is 50, key is test_row_0/C:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:40,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742362_1538 (size=12001) 2024-11-19T05:26:40,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:40,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:41,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:41,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:41,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ef72464a19c48de9adafc263d066afa 2024-11-19T05:26:41,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/8674b72a08b246ccb488d7dddad41c4f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f 2024-11-19T05:26:41,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f, entries=150, sequenceid=95, filesize=30.2 K 2024-11-19T05:26:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/4ce7df5b94e644a2a9545382cd63367e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e 2024-11-19T05:26:41,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T05:26:41,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ef72464a19c48de9adafc263d066afa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa 2024-11-19T05:26:41,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa, entries=150, sequenceid=95, filesize=11.7 K 2024-11-19T05:26:41,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 3086702af0f4dab24249af64f43614f1 in 1681ms, sequenceid=95, compaction requested=true 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:41,240 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:41,240 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:41,241 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:41,240 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:41,241 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:41,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:41,241 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,241 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/f295b2eae5fb42859f1e5094aa1aff5d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.3 K 2024-11-19T05:26:41,242 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:41,242 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:41,242 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f295b2eae5fb42859f1e5094aa1aff5d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:41,242 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,242 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=90.8 K 2024-11-19T05:26:41,242 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,242 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f] 2024-11-19T05:26:41,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 0246218150cb455fb7ba0cfbcd94d8d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731993997278 2024-11-19T05:26:41,243 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9f3d933d2834057864c0f6cbaab1a7b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:41,243 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce7df5b94e644a2a9545382cd63367e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:41,243 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6e045d8015941a6a99de4d2977027ef, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731993997278 2024-11-19T05:26:41,243 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8674b72a08b246ccb488d7dddad41c4f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:41,253 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:41,255 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:41,256 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/14b0946fb3834830b0d3e603bc87dfbf is 50, key is test_row_0/B:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:41,261 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411198b46f659aea14cde80173de3fc263916_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:41,263 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411198b46f659aea14cde80173de3fc263916_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:41,264 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198b46f659aea14cde80173de3fc263916_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742363_1539 (size=12207) 2024-11-19T05:26:41,276 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/14b0946fb3834830b0d3e603bc87dfbf as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/14b0946fb3834830b0d3e603bc87dfbf 2024-11-19T05:26:41,281 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 14b0946fb3834830b0d3e603bc87dfbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:41,281 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:41,281 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994001240; duration=0sec 2024-11-19T05:26:41,281 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:41,282 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:41,282 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:41,283 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:41,283 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:41,283 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,283 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7d158bfa66b74d7884f2f8c313d2a85a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.3 K 2024-11-19T05:26:41,283 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d158bfa66b74d7884f2f8c313d2a85a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731993995090 2024-11-19T05:26:41,284 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b516cf07664ac5b1c635109a1ee238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731993997278 2024-11-19T05:26:41,284 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ef72464a19c48de9adafc263d066afa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:41,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742364_1540 (size=4469) 2024-11-19T05:26:41,294 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#458 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:41,295 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/20fcb2b59463435dacbc3e5b8d5f5596 is 175, key is test_row_0/A:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:41,302 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#460 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:41,303 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/5432d08bc87d4f30bbed57d66e4ec08e is 50, key is test_row_0/C:col10/1731993999558/Put/seqid=0 2024-11-19T05:26:41,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-19T05:26:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:41,314 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T05:26:41,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742365_1541 (size=31161) 2024-11-19T05:26:41,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742366_1542 (size=12207) 2024-11-19T05:26:41,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9ec519df38f4e05893cf053b00d0403_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731993999570/Put/seqid=0 2024-11-19T05:26:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742367_1543 (size=12154) 2024-11-19T05:26:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:41,358 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119b9ec519df38f4e05893cf053b00d0403_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9ec519df38f4e05893cf053b00d0403_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:41,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c398e0ca5f6a42b98dbe43620953f0e2, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:41,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c398e0ca5f6a42b98dbe43620953f0e2 is 175, key is test_row_0/A:col10/1731993999570/Put/seqid=0 2024-11-19T05:26:41,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742368_1544 (size=30955) 2024-11-19T05:26:41,365 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c398e0ca5f6a42b98dbe43620953f0e2 2024-11-19T05:26:41,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/05f21bce907c48bda31e4c5d5c557106 is 50, key is test_row_0/B:col10/1731993999570/Put/seqid=0 2024-11-19T05:26:41,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742369_1545 (size=12001) 2024-11-19T05:26:41,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:41,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:41,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994061517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994061627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,738 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/20fcb2b59463435dacbc3e5b8d5f5596 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596 2024-11-19T05:26:41,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994061737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994061738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994061738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,739 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:41,739 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:41,739 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,743 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/5432d08bc87d4f30bbed57d66e4ec08e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5432d08bc87d4f30bbed57d66e4ec08e 2024-11-19T05:26:41,744 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 20fcb2b59463435dacbc3e5b8d5f5596(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:41,744 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:41,744 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994001240; duration=0sec 2024-11-19T05:26:41,744 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:41,744 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:41,749 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 5432d08bc87d4f30bbed57d66e4ec08e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:41,749 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:41,749 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994001240; duration=0sec 2024-11-19T05:26:41,749 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:41,749 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:41,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994061747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,750 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:41,776 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/05f21bce907c48bda31e4c5d5c557106 2024-11-19T05:26:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:41,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ac4a6eaf0be4572b74131149389cd62 is 50, key is test_row_0/C:col10/1731993999570/Put/seqid=0 2024-11-19T05:26:41,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742370_1546 (size=12001) 2024-11-19T05:26:41,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:41,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994061837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:41,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:42,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:42,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994062147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:42,222 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ac4a6eaf0be4572b74131149389cd62 2024-11-19T05:26:42,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c398e0ca5f6a42b98dbe43620953f0e2 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2 2024-11-19T05:26:42,229 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2, entries=150, sequenceid=117, filesize=30.2 K 2024-11-19T05:26:42,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/05f21bce907c48bda31e4c5d5c557106 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106 2024-11-19T05:26:42,235 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T05:26:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7ac4a6eaf0be4572b74131149389cd62 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62 2024-11-19T05:26:42,239 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62, entries=150, sequenceid=117, filesize=11.7 K 2024-11-19T05:26:42,239 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3086702af0f4dab24249af64f43614f1 in 925ms, sequenceid=117, compaction requested=false 2024-11-19T05:26:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:42,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:42,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-19T05:26:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-19T05:26:42,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-19T05:26:42,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6080 sec 2024-11-19T05:26:42,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.6150 sec 2024-11-19T05:26:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:42,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:42,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:26:42,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912db1cd5e6414ab1be20025033e8c7ff_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:42,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742371_1547 (size=14744) 2024-11-19T05:26:42,669 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:42,689 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111912db1cd5e6414ab1be20025033e8c7ff_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912db1cd5e6414ab1be20025033e8c7ff_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:42,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/648a589f8fb042b4aa87cc58c1432bc7, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:42,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/648a589f8fb042b4aa87cc58c1432bc7 is 175, key is test_row_0/A:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:42,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742372_1548 (size=39699) 2024-11-19T05:26:42,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/648a589f8fb042b4aa87cc58c1432bc7 2024-11-19T05:26:42,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/6881dbc64df24fd086adcbed4135b27b is 50, key is test_row_0/B:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:42,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742373_1549 (size=12101) 2024-11-19T05:26:42,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994062756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,758 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:42,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994062867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994063077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,082 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/6881dbc64df24fd086adcbed4135b27b 2024-11-19T05:26:43,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fd52060022a74f19978fa1d517f4c1e5 is 50, key is test_row_0/C:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:43,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742374_1550 (size=12101) 2024-11-19T05:26:43,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fd52060022a74f19978fa1d517f4c1e5 2024-11-19T05:26:43,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/648a589f8fb042b4aa87cc58c1432bc7 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7 2024-11-19T05:26:43,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7, entries=200, sequenceid=135, filesize=38.8 K 2024-11-19T05:26:43,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/6881dbc64df24fd086adcbed4135b27b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b 2024-11-19T05:26:43,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b, entries=150, sequenceid=135, filesize=11.8 K 2024-11-19T05:26:43,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fd52060022a74f19978fa1d517f4c1e5 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5 2024-11-19T05:26:43,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5, entries=150, sequenceid=135, filesize=11.8 K 2024-11-19T05:26:43,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3086702af0f4dab24249af64f43614f1 in 510ms, sequenceid=135, compaction requested=true 2024-11-19T05:26:43,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:43,170 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:43,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-19T05:26:43,171 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:43,171 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:43,171 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,171 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:43,171 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=99.4 K 2024-11-19T05:26:43,171 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,171 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7] 2024-11-19T05:26:43,171 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20fcb2b59463435dacbc3e5b8d5f5596, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:43,172 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting c398e0ca5f6a42b98dbe43620953f0e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731993999570 2024-11-19T05:26:43,172 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 648a589f8fb042b4aa87cc58c1432bc7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:43,172 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:43,172 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:43,172 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,172 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/14b0946fb3834830b0d3e603bc87dfbf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.5 K 2024-11-19T05:26:43,172 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b0946fb3834830b0d3e603bc87dfbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:43,173 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 05f21bce907c48bda31e4c5d5c557106, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731993999570 2024-11-19T05:26:43,173 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 6881dbc64df24fd086adcbed4135b27b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:43,179 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:43,179 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/9d5c0e79061f463f8959e04417120f1b is 50, key is test_row_0/B:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:43,182 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,185 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119a395fc7b65ff41529d452a125aef0538_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,187 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119a395fc7b65ff41529d452a125aef0538_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,187 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a395fc7b65ff41529d452a125aef0538_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742375_1551 (size=12409) 2024-11-19T05:26:43,199 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/9d5c0e79061f463f8959e04417120f1b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9d5c0e79061f463f8959e04417120f1b 2024-11-19T05:26:43,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742376_1552 (size=4469) 2024-11-19T05:26:43,203 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 9d5c0e79061f463f8959e04417120f1b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:43,204 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:43,204 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994003170; duration=0sec 2024-11-19T05:26:43,204 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:43,204 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:43,204 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:43,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:43,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:43,205 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,205 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5432d08bc87d4f30bbed57d66e4ec08e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.5 K 2024-11-19T05:26:43,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5432d08bc87d4f30bbed57d66e4ec08e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731993999446 2024-11-19T05:26:43,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ac4a6eaf0be4572b74131149389cd62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731993999570 2024-11-19T05:26:43,205 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fd52060022a74f19978fa1d517f4c1e5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:43,212 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:43,212 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/adbb9af7831941299da671e0a6bfdbdd is 50, key is test_row_0/C:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:43,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742377_1553 (size=12409) 2024-11-19T05:26:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:43,390 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:43,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-19T05:26:43,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119405e04f6abcf4e92be39d9dd9a1f40fc_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994002746/Put/seqid=0 2024-11-19T05:26:43,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742378_1554 (size=14794) 2024-11-19T05:26:43,424 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:43,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119405e04f6abcf4e92be39d9dd9a1f40fc_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119405e04f6abcf4e92be39d9dd9a1f40fc_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:43,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/15b578acc63440c98f42e7f7e2d1eef3, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/15b578acc63440c98f42e7f7e2d1eef3 is 175, key is test_row_0/A:col10/1731994002746/Put/seqid=0 2024-11-19T05:26:43,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742379_1555 (size=39749) 2024-11-19T05:26:43,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/15b578acc63440c98f42e7f7e2d1eef3 2024-11-19T05:26:43,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/7af9b872939148059de55d9e7f8ce8fd is 50, key is test_row_0/B:col10/1731994002746/Put/seqid=0 2024-11-19T05:26:43,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742380_1556 (size=12151) 2024-11-19T05:26:43,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/7af9b872939148059de55d9e7f8ce8fd 2024-11-19T05:26:43,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994063448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f4825d637c1d461caa59e97fcc87f245 is 50, key is test_row_0/C:col10/1731994002746/Put/seqid=0 2024-11-19T05:26:43,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742381_1557 (size=12151) 2024-11-19T05:26:43,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f4825d637c1d461caa59e97fcc87f245 2024-11-19T05:26:43,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/15b578acc63440c98f42e7f7e2d1eef3 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3 2024-11-19T05:26:43,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3, entries=200, sequenceid=157, filesize=38.8 K 2024-11-19T05:26:43,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/7af9b872939148059de55d9e7f8ce8fd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd 2024-11-19T05:26:43,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T05:26:43,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f4825d637c1d461caa59e97fcc87f245 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245 2024-11-19T05:26:43,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245, entries=150, sequenceid=157, filesize=11.9 K 2024-11-19T05:26:43,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 3086702af0f4dab24249af64f43614f1 in 90ms, sequenceid=157, compaction requested=false 2024-11-19T05:26:43,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:43,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:43,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-19T05:26:43,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191f433fe8b3514c5d8f51098d2a378e3f_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:43,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742382_1558 (size=14794) 2024-11-19T05:26:43,603 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:43,604 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/828821a9f9c548bdbe8d631c45750abd is 175, key is test_row_0/A:col10/1731994002658/Put/seqid=0 2024-11-19T05:26:43,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742383_1559 (size=31363) 2024-11-19T05:26:43,626 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/adbb9af7831941299da671e0a6bfdbdd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/adbb9af7831941299da671e0a6bfdbdd 2024-11-19T05:26:43,629 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into adbb9af7831941299da671e0a6bfdbdd(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:43,629 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:43,629 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994003170; duration=0sec 2024-11-19T05:26:43,629 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:43,629 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:43,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994063631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994063747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994063757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994063757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994063757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,761 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:43,762 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_0' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,762 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994063767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,769 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_2' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-19T05:26:43,797 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:43,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-19T05:26:43,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:43,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:43,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:43,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T05:26:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:43,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:43,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:43,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:43,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:43,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994063957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:43,960 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:43,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:43,980 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411191f433fe8b3514c5d8f51098d2a378e3f_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191f433fe8b3514c5d8f51098d2a378e3f_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:43,981 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/871880031907402a8764c2b8807cd451, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:43,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/871880031907402a8764c2b8807cd451 is 175, key is test_row_0/A:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742384_1560 (size=39749) 2024-11-19T05:26:44,022 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/828821a9f9c548bdbe8d631c45750abd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd 2024-11-19T05:26:44,030 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 828821a9f9c548bdbe8d631c45750abd(size=30.6 K), total size for store is 69.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:44,031 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:44,031 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994003169; duration=0sec 2024-11-19T05:26:44,031 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:44,031 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:44,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T05:26:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:44,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:44,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T05:26:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:44,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994064268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,270 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:44,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T05:26:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/871880031907402a8764c2b8807cd451 2024-11-19T05:26:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:44,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2c2c5b6d741741ada6808c9720de1e81 is 50, key is test_row_0/B:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742385_1561 (size=12151) 2024-11-19T05:26:44,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2c2c5b6d741741ada6808c9720de1e81 2024-11-19T05:26:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:44,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/984d06ef215f44c5a4b8be39e334acbc is 50, key is test_row_0/C:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742386_1562 (size=12151) 2024-11-19T05:26:44,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/984d06ef215f44c5a4b8be39e334acbc 2024-11-19T05:26:44,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/871880031907402a8764c2b8807cd451 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451 2024-11-19T05:26:44,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451, entries=200, sequenceid=173, filesize=38.8 K 2024-11-19T05:26:44,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2c2c5b6d741741ada6808c9720de1e81 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81 2024-11-19T05:26:44,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T05:26:44,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/984d06ef215f44c5a4b8be39e334acbc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc 2024-11-19T05:26:44,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc, entries=150, sequenceid=173, filesize=11.9 K 2024-11-19T05:26:44,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3086702af0f4dab24249af64f43614f1 in 914ms, sequenceid=173, compaction requested=true 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:44,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:44,473 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:44,473 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:44,474 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:44,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:44,474 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:44,474 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:44,474 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,474 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,474 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9d5c0e79061f463f8959e04417120f1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.9 K 2024-11-19T05:26:44,474 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=108.3 K 2024-11-19T05:26:44,474 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451] 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 828821a9f9c548bdbe8d631c45750abd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5c0e79061f463f8959e04417120f1b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af9b872939148059de55d9e7f8ce8fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731994002746 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b578acc63440c98f42e7f7e2d1eef3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731994002730 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c2c5b6d741741ada6808c9720de1e81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:44,475 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 871880031907402a8764c2b8807cd451, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:44,481 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:44,482 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#476 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:44,482 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/87c4a1f6ec9b4d4faa2b19211a456f68 is 50, key is test_row_0/B:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,492 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411197a4735471f5447d4baa06b80c96d2000_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:44,494 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411197a4735471f5447d4baa06b80c96d2000_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:44,494 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411197a4735471f5447d4baa06b80c96d2000_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:44,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742387_1563 (size=12561) 2024-11-19T05:26:44,501 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/87c4a1f6ec9b4d4faa2b19211a456f68 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/87c4a1f6ec9b4d4faa2b19211a456f68 2024-11-19T05:26:44,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742388_1564 (size=4469) 2024-11-19T05:26:44,511 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 87c4a1f6ec9b4d4faa2b19211a456f68(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:44,511 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:44,511 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994004473; duration=0sec 2024-11-19T05:26:44,511 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:44,511 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:44,511 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:44,513 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#477 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:44,513 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/73c88cb17c384c52a4b1b130ba2f24f6 is 175, key is test_row_0/A:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,513 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:44,514 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:44,514 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,514 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/adbb9af7831941299da671e0a6bfdbdd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=35.9 K 2024-11-19T05:26:44,514 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting adbb9af7831941299da671e0a6bfdbdd, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731994001500 2024-11-19T05:26:44,515 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f4825d637c1d461caa59e97fcc87f245, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731994002746 2024-11-19T05:26:44,516 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 984d06ef215f44c5a4b8be39e334acbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:44,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742389_1565 (size=31515) 2024-11-19T05:26:44,523 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:44,523 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/51081e6db14b4d5cb0c01d725884f21e is 50, key is test_row_0/C:col10/1731994003442/Put/seqid=0 2024-11-19T05:26:44,524 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/73c88cb17c384c52a4b1b130ba2f24f6 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6 2024-11-19T05:26:44,528 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 73c88cb17c384c52a4b1b130ba2f24f6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:44,528 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:44,528 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994004473; duration=0sec 2024-11-19T05:26:44,528 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:44,528 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:44,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742390_1566 (size=12561) 2024-11-19T05:26:44,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-19T05:26:44,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:44,563 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-19T05:26:44,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f602c397fef44ec839a69cd1f678347_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994003616/Put/seqid=0 2024-11-19T05:26:44,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742391_1567 (size=12304) 2024-11-19T05:26:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:44,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:44,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:44,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994064823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,826 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994064937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:44,944 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/51081e6db14b4d5cb0c01d725884f21e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/51081e6db14b4d5cb0c01d725884f21e 2024-11-19T05:26:44,947 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 51081e6db14b4d5cb0c01d725884f21e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:44,947 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:44,947 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994004473; duration=0sec 2024-11-19T05:26:44,947 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:44,948 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:44,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:44,974 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411193f602c397fef44ec839a69cd1f678347_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f602c397fef44ec839a69cd1f678347_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:44,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/93e4072e24f1400cb8041aaf5fb86075, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:44,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/93e4072e24f1400cb8041aaf5fb86075 is 175, key is test_row_0/A:col10/1731994003616/Put/seqid=0 2024-11-19T05:26:44,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742392_1568 (size=31105) 2024-11-19T05:26:45,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994065147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:45,149 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:45,378 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/93e4072e24f1400cb8041aaf5fb86075 2024-11-19T05:26:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2cc7a1ef6213420da01c382305831007 is 50, key is test_row_0/B:col10/1731994003616/Put/seqid=0 2024-11-19T05:26:45,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742393_1569 (size=12151) 2024-11-19T05:26:45,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:45,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994065457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:45,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:45,786 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2cc7a1ef6213420da01c382305831007 2024-11-19T05:26:45,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/a4655607388341e59cc1be712a0ed982 is 50, key is test_row_0/C:col10/1731994003616/Put/seqid=0 2024-11-19T05:26:45,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742394_1570 (size=12151) 2024-11-19T05:26:45,803 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/a4655607388341e59cc1be712a0ed982 2024-11-19T05:26:45,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/93e4072e24f1400cb8041aaf5fb86075 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075 2024-11-19T05:26:45,809 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075, entries=150, sequenceid=197, filesize=30.4 K 2024-11-19T05:26:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2cc7a1ef6213420da01c382305831007 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007 2024-11-19T05:26:45,812 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007, entries=150, sequenceid=197, filesize=11.9 K 2024-11-19T05:26:45,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/a4655607388341e59cc1be712a0ed982 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982 2024-11-19T05:26:45,815 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982, entries=150, sequenceid=197, filesize=11.9 K 2024-11-19T05:26:45,816 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3086702af0f4dab24249af64f43614f1 in 1253ms, sequenceid=197, compaction requested=false 2024-11-19T05:26:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-19T05:26:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-19T05:26:45,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-19T05:26:45,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-11-19T05:26:45,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.0210 sec 2024-11-19T05:26:45,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-19T05:26:45,948 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:45,948 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-19T05:26:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:45,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:45,950 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:45,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:45,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:45,970 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:45,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-19T05:26:45,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119acc07389bcb743b490cfe9fa5baeebd3_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:45,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742395_1571 (size=14794) 2024-11-19T05:26:46,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:46,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994066034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:46,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:46,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:46,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994066147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,150 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:46,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:46,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994066357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:46,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:46,382 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119acc07389bcb743b490cfe9fa5baeebd3_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acc07389bcb743b490cfe9fa5baeebd3_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:46,382 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/af86ac38ce9045909472e2e9b678f949, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:46,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/af86ac38ce9045909472e2e9b678f949 is 175, key is test_row_0/A:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742396_1572 (size=39749) 2024-11-19T05:26:46,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:46,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994066667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:46,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,795 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/af86ac38ce9045909472e2e9b678f949 2024-11-19T05:26:46,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/ed69f237ef064c279de7276f8241a84e is 50, key is test_row_0/B:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:46,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742397_1573 (size=12151) 2024-11-19T05:26:46,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:46,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:46,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:47,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994067188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:47,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/ed69f237ef064c279de7276f8241a84e 2024-11-19T05:26:47,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/d6cdaaa4ec69484a95699dcbb18569b1 is 50, key is test_row_0/C:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:47,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742398_1574 (size=12151) 2024-11-19T05:26:47,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/d6cdaaa4ec69484a95699dcbb18569b1 2024-11-19T05:26:47,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/af86ac38ce9045909472e2e9b678f949 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949 2024-11-19T05:26:47,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949, entries=200, sequenceid=215, filesize=38.8 K 2024-11-19T05:26:47,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/ed69f237ef064c279de7276f8241a84e as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e 2024-11-19T05:26:47,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T05:26:47,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/d6cdaaa4ec69484a95699dcbb18569b1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1 2024-11-19T05:26:47,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1, entries=150, sequenceid=215, filesize=11.9 K 2024-11-19T05:26:47,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3086702af0f4dab24249af64f43614f1 in 1281ms, sequenceid=215, compaction requested=true 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:47,251 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:47,251 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:47,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:47,252 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:47,252 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:47,252 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:47,252 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:47,252 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,252 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,252 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=100.0 K 2024-11-19T05:26:47,252 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/87c4a1f6ec9b4d4faa2b19211a456f68, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.0 K 2024-11-19T05:26:47,252 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,252 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949] 2024-11-19T05:26:47,253 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c4a1f6ec9b4d4faa2b19211a456f68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:47,253 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73c88cb17c384c52a4b1b130ba2f24f6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:47,253 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cc7a1ef6213420da01c382305831007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731994003608 2024-11-19T05:26:47,254 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93e4072e24f1400cb8041aaf5fb86075, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731994003608 2024-11-19T05:26:47,254 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ed69f237ef064c279de7276f8241a84e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:47,254 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting af86ac38ce9045909472e2e9b678f949, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:47,260 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:47,260 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2406ed06eacc4a6694892dc4105b257c is 50, key is test_row_0/B:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:47,268 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:47,281 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194a5a90a662e345bd9bd8df46d7633363_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:47,282 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194a5a90a662e345bd9bd8df46d7633363_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:47,282 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194a5a90a662e345bd9bd8df46d7633363_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:47,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742399_1575 (size=12663) 2024-11-19T05:26:47,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742400_1576 (size=4469) 2024-11-19T05:26:47,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-19T05:26:47,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,323 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-19T05:26:47,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198508335a1c02464e92b51da2c6a7d3ac_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994006024/Put/seqid=0 2024-11-19T05:26:47,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742401_1577 (size=12304) 2024-11-19T05:26:47,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,350 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411198508335a1c02464e92b51da2c6a7d3ac_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198508335a1c02464e92b51da2c6a7d3ac_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:47,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/ad8e173175e445c2bdd9438295777148, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:47,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/ad8e173175e445c2bdd9438295777148 is 175, key is test_row_0/A:col10/1731994006024/Put/seqid=0 2024-11-19T05:26:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742402_1578 (size=31105) 2024-11-19T05:26:47,694 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/2406ed06eacc4a6694892dc4105b257c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2406ed06eacc4a6694892dc4105b257c 2024-11-19T05:26:47,698 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 2406ed06eacc4a6694892dc4105b257c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:47,698 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:47,698 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994007251; duration=0sec 2024-11-19T05:26:47,698 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:47,698 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:47,698 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:47,699 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:47,699 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:47,699 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,699 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/51081e6db14b4d5cb0c01d725884f21e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.0 K 2024-11-19T05:26:47,699 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 51081e6db14b4d5cb0c01d725884f21e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731994003436 2024-11-19T05:26:47,699 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a4655607388341e59cc1be712a0ed982, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731994003608 2024-11-19T05:26:47,700 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting d6cdaaa4ec69484a95699dcbb18569b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:47,704 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:47,704 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7489fc013d014db1b9c27965ef90ec15 is 50, key is test_row_0/C:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:47,705 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:47,705 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/77e710f196c24ac188d0517e6099bc0a is 175, key is test_row_0/A:col10/1731994004813/Put/seqid=0 2024-11-19T05:26:47,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742403_1579 (size=12663) 2024-11-19T05:26:47,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742404_1580 (size=31617) 2024-11-19T05:26:47,713 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/7489fc013d014db1b9c27965ef90ec15 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7489fc013d014db1b9c27965ef90ec15 2024-11-19T05:26:47,720 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 7489fc013d014db1b9c27965ef90ec15(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:47,720 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:47,720 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994007251; duration=0sec 2024-11-19T05:26:47,720 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:47,720 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:47,755 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/ad8e173175e445c2bdd9438295777148 2024-11-19T05:26:47,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/16d3f4d79ec44f59b28d4721b7b9667d is 50, key is test_row_0/B:col10/1731994006024/Put/seqid=0 2024-11-19T05:26:47,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742405_1581 (size=12151) 2024-11-19T05:26:47,765 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/16d3f4d79ec44f59b28d4721b7b9667d 2024-11-19T05:26:47,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f31a501f8990425ba9dbafc5ce05a076 is 50, key is test_row_0/C:col10/1731994006024/Put/seqid=0 2024-11-19T05:26:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742406_1582 (size=12151) 2024-11-19T05:26:47,775 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f31a501f8990425ba9dbafc5ce05a076 2024-11-19T05:26:47,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/ad8e173175e445c2bdd9438295777148 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148 2024-11-19T05:26:47,783 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148, entries=150, sequenceid=234, filesize=30.4 K 2024-11-19T05:26:47,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/16d3f4d79ec44f59b28d4721b7b9667d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d 2024-11-19T05:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,786 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d, entries=150, sequenceid=234, filesize=11.9 K 2024-11-19T05:26:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/f31a501f8990425ba9dbafc5ce05a076 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076 2024-11-19T05:26:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076, entries=150, sequenceid=234, filesize=11.9 K 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 3086702af0f4dab24249af64f43614f1 in 472ms, sequenceid=234, compaction requested=false 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-19T05:26:47,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8470 sec 2024-11-19T05:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8520 sec 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:47,821 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:47,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:47,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:47,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a36f033514634a67b5321c5fdc2f7423_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:47,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742408_1584 (size=24758) 2024-11-19T05:26:47,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994067888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:47,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994067888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:47,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994067889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:47,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994067895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:47,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994067997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994067997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994068007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994068007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-19T05:26:48,098 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:48,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:48,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-19T05:26:48,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:48,100 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:48,101 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:48,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:48,113 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/77e710f196c24ac188d0517e6099bc0a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a 2024-11-19T05:26:48,117 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 77e710f196c24ac188d0517e6099bc0a(size=30.9 K), total size for store is 61.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:48,117 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:48,117 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994007251; duration=0sec 2024-11-19T05:26:48,117 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:48,117 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:48,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994068197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,199 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to f990f635ffd0,38449,1731993838792 for 'test_row_1' in 3086702af0f4dab24249af64f43614f1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-19T05:26:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994068207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994068207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994068217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994068218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,239 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:48,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119a36f033514634a67b5321c5fdc2f7423_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a36f033514634a67b5321c5fdc2f7423_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:48,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/5f0ad32b02204bb1acec604562e7d38f, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:48,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/5f0ad32b02204bb1acec604562e7d38f is 175, key is test_row_0/A:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:48,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742407_1583 (size=74394) 2024-11-19T05:26:48,245 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/5f0ad32b02204bb1acec604562e7d38f 2024-11-19T05:26:48,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/9086bab4b67048ccad44139667e2de5a is 50, key is test_row_0/B:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:48,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T05:26:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742409_1585 (size=12151) 2024-11-19T05:26:48,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T05:26:48,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994068517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994068517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994068527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994068527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:48,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T05:26:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:48,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/9086bab4b67048ccad44139667e2de5a 2024-11-19T05:26:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fb7f560c65e447abb8a3ce8c373cb9e4 is 50, key is test_row_0/C:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:48,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742410_1586 (size=12151) 2024-11-19T05:26:48,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fb7f560c65e447abb8a3ce8c373cb9e4 2024-11-19T05:26:48,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/5f0ad32b02204bb1acec604562e7d38f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f 2024-11-19T05:26:48,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f, entries=400, sequenceid=247, filesize=72.7 K 2024-11-19T05:26:48,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/9086bab4b67048ccad44139667e2de5a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a 2024-11-19T05:26:48,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a, entries=150, sequenceid=247, filesize=11.9 K 2024-11-19T05:26:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fb7f560c65e447abb8a3ce8c373cb9e4 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4 2024-11-19T05:26:48,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4, entries=150, sequenceid=247, filesize=11.9 K 2024-11-19T05:26:48,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3086702af0f4dab24249af64f43614f1 in 868ms, sequenceid=247, compaction requested=true 2024-11-19T05:26:48,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:48,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:48,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:48,689 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:48,689 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:48,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137116 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:48,691 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,691 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,691 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2406ed06eacc4a6694892dc4105b257c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.1 K 2024-11-19T05:26:48,691 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=133.9 K 2024-11-19T05:26:48,691 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f] 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 2406ed06eacc4a6694892dc4105b257c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:48,691 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77e710f196c24ac188d0517e6099bc0a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:48,692 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 16d3f4d79ec44f59b28d4721b7b9667d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731994006024 2024-11-19T05:26:48,692 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad8e173175e445c2bdd9438295777148, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731994006024 2024-11-19T05:26:48,692 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 9086bab4b67048ccad44139667e2de5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007816 2024-11-19T05:26:48,692 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f0ad32b02204bb1acec604562e7d38f, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007805 2024-11-19T05:26:48,705 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:48,706 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:48,707 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/692af004793e4fc7a9e6c3549212926a is 50, key is test_row_0/B:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:48,707 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241119754dcbef3e2d49e7b0865c965e9094b9_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:48,709 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241119754dcbef3e2d49e7b0865c965e9094b9_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:48,709 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119754dcbef3e2d49e7b0865c965e9094b9_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:48,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-19T05:26:48,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,715 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-19T05:26:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742411_1587 (size=12765) 2024-11-19T05:26:48,725 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/692af004793e4fc7a9e6c3549212926a as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/692af004793e4fc7a9e6c3549212926a 2024-11-19T05:26:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:48,729 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 692af004793e4fc7a9e6c3549212926a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:48,729 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:48,729 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994008689; duration=0sec 2024-11-19T05:26:48,729 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:48,729 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:48,729 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:48,730 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:48,730 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:48,730 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:48,730 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7489fc013d014db1b9c27965ef90ec15, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.1 K 2024-11-19T05:26:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742412_1588 (size=4469) 2024-11-19T05:26:48,732 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 7489fc013d014db1b9c27965ef90ec15, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731994004813 2024-11-19T05:26:48,733 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting f31a501f8990425ba9dbafc5ce05a076, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731994006024 2024-11-19T05:26:48,733 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fb7f560c65e447abb8a3ce8c373cb9e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007816 2024-11-19T05:26:48,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119353b433252d64cfbba324db6cc24b8cd_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994007893/Put/seqid=0 2024-11-19T05:26:48,738 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:48,738 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/a05dcfb872d54069bce59dac5037a5bd is 50, key is test_row_0/C:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:48,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742413_1589 (size=12454) 2024-11-19T05:26:48,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742414_1590 (size=12765) 2024-11-19T05:26:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:49,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994069037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994069038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,042 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994069039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994069040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,133 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#495 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:49,134 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/36642fff141f495eb8c976752380c582 is 175, key is test_row_0/A:col10/1731994007820/Put/seqid=0 2024-11-19T05:26:49,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742415_1591 (size=31719) 2024-11-19T05:26:49,140 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/36642fff141f495eb8c976752380c582 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582 2024-11-19T05:26:49,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:49,144 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 36642fff141f495eb8c976752380c582(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:49,144 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:49,144 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994008689; duration=0sec 2024-11-19T05:26:49,144 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:49,144 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:49,149 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119353b433252d64cfbba324db6cc24b8cd_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119353b433252d64cfbba324db6cc24b8cd_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:49,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994069147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,151 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/a05dcfb872d54069bce59dac5037a5bd as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a05dcfb872d54069bce59dac5037a5bd 2024-11-19T05:26:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1f893b1f43474e69ba087117b8f624b1, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:49,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1f893b1f43474e69ba087117b8f624b1 is 175, key is test_row_0/A:col10/1731994007893/Put/seqid=0 2024-11-19T05:26:49,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994069148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,161 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into a05dcfb872d54069bce59dac5037a5bd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:49,161 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:49,161 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994008690; duration=0sec 2024-11-19T05:26:49,161 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:49,161 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:49,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742416_1592 (size=31255) 2024-11-19T05:26:49,164 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1f893b1f43474e69ba087117b8f624b1 2024-11-19T05:26:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/679c7902714a4281a233cd23e054cf5f is 50, key is test_row_0/B:col10/1731994007893/Put/seqid=0 2024-11-19T05:26:49,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742417_1593 (size=12301) 2024-11-19T05:26:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994069357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994069357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994069357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994069357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,587 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/679c7902714a4281a233cd23e054cf5f 2024-11-19T05:26:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/cb59262991814ccab85c35cb0f441049 is 50, key is test_row_0/C:col10/1731994007893/Put/seqid=0 2024-11-19T05:26:49,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742418_1594 (size=12301) 2024-11-19T05:26:49,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994069668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,669 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994069667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994069668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:49,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994069668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:49,995 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/cb59262991814ccab85c35cb0f441049 2024-11-19T05:26:49,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/1f893b1f43474e69ba087117b8f624b1 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1 2024-11-19T05:26:50,001 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1, entries=150, sequenceid=273, filesize=30.5 K 2024-11-19T05:26:50,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/679c7902714a4281a233cd23e054cf5f as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f 2024-11-19T05:26:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,004 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f, entries=150, sequenceid=273, filesize=12.0 K 2024-11-19T05:26:50,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/cb59262991814ccab85c35cb0f441049 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049 2024-11-19T05:26:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049, entries=150, sequenceid=273, filesize=12.0 K 2024-11-19T05:26:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,008 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3086702af0f4dab24249af64f43614f1 in 1293ms, sequenceid=273, compaction requested=false 2024-11-19T05:26:50,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-19T05:26:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-19T05:26:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-19T05:26:50,015 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-19T05:26:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.9180 sec 2024-11-19T05:26:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,182 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:50,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-19T05:26:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:50,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed88135217a64ec89926563ae64b824c_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742419_1595 (size=14994) 2024-11-19T05:26:50,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994070220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994070221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994070224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994070225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994070225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-19T05:26:50,257 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:50,257 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-19T05:26:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-19T05:26:50,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-19T05:26:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:50,260 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-19T05:26:50,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-19T05:26:50,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994070327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994070337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994070338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994070338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994070338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:50,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994070537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994070547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994070547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994070547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994070547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:50,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:50,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:50,610 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:50,613 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119ed88135217a64ec89926563ae64b824c_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed88135217a64ec89926563ae64b824c_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:50,615 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/234799519b0549c59596173960d42721, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:50,616 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/234799519b0549c59596173960d42721 is 175, key is test_row_0/A:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:50,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742420_1596 (size=39949) 2024-11-19T05:26:50,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994070847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994070857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:50,861 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:50,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:50,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:51,024 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/234799519b0549c59596173960d42721 2024-11-19T05:26:51,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/a05e4da19f9d423da2929be6a0aa829c is 50, key is test_row_0/B:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:51,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742421_1597 (size=12301) 2024-11-19T05:26:51,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:51,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:51,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T05:26:51,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994071357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994071367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:51,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994071367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994071367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:51,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994071377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:51,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/a05e4da19f9d423da2929be6a0aa829c 2024-11-19T05:26:51,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fe0e56115b9d4bd5b1128cd15dcc8b22 is 50, key is test_row_0/C:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:51,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742422_1598 (size=12301) 2024-11-19T05:26:51,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fe0e56115b9d4bd5b1128cd15dcc8b22 2024-11-19T05:26:51,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/234799519b0549c59596173960d42721 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721 2024-11-19T05:26:51,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721, entries=200, sequenceid=289, filesize=39.0 K 2024-11-19T05:26:51,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/a05e4da19f9d423da2929be6a0aa829c as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c 2024-11-19T05:26:51,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c, entries=150, sequenceid=289, filesize=12.0 K 2024-11-19T05:26:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/fe0e56115b9d4bd5b1128cd15dcc8b22 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22 2024-11-19T05:26:51,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22, entries=150, sequenceid=289, filesize=12.0 K 2024-11-19T05:26:51,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3086702af0f4dab24249af64f43614f1 in 1284ms, sequenceid=289, compaction requested=true 2024-11-19T05:26:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:51,466 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:51,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:51,467 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:51,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:51,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:51,467 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:51,467 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:51,467 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:51,467 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:51,467 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,467 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,467 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/692af004793e4fc7a9e6c3549212926a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.5 K 2024-11-19T05:26:51,467 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=100.5 K 2024-11-19T05:26:51,467 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,468 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721] 2024-11-19T05:26:51,468 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 692af004793e4fc7a9e6c3549212926a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007816 2024-11-19T05:26:51,468 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36642fff141f495eb8c976752380c582, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007816 2024-11-19T05:26:51,469 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f893b1f43474e69ba087117b8f624b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731994007862 2024-11-19T05:26:51,469 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 679c7902714a4281a233cd23e054cf5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731994007862 2024-11-19T05:26:51,469 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 234799519b0549c59596173960d42721, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:51,469 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a05e4da19f9d423da2929be6a0aa829c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:51,478 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:51,479 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/b710c51731ff4d88b28f578a68174f85 is 50, key is test_row_0/B:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:51,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38449 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-19T05:26:51,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,483 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-19T05:26:51,493 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:51,501 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411195456a61c310245c2b9f629b01fad828c_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:51,502 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411195456a61c310245c2b9f629b01fad828c_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:51,502 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411195456a61c310245c2b9f629b01fad828c_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:51,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119db5a05cadc334fc2b9918551ff0b35e2_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994010211/Put/seqid=0 2024-11-19T05:26:51,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742423_1599 (size=13017) 2024-11-19T05:26:51,516 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/b710c51731ff4d88b28f578a68174f85 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/b710c51731ff4d88b28f578a68174f85 2024-11-19T05:26:51,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742424_1600 (size=4469) 2024-11-19T05:26:51,521 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#504 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:51,521 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/0777b789c3b84d09a444c46232116317 is 175, key is test_row_0/A:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:51,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742425_1601 (size=12454) 2024-11-19T05:26:51,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:51,526 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into b710c51731ff4d88b28f578a68174f85(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:51,526 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:51,526 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994011466; duration=0sec 2024-11-19T05:26:51,526 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:51,526 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:51,526 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:51,527 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:51,527 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:51,527 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:51,527 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a05dcfb872d54069bce59dac5037a5bd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.5 K 2024-11-19T05:26:51,528 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting a05dcfb872d54069bce59dac5037a5bd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731994007816 2024-11-19T05:26:51,530 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119db5a05cadc334fc2b9918551ff0b35e2_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db5a05cadc334fc2b9918551ff0b35e2_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:51,530 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting cb59262991814ccab85c35cb0f441049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731994007862 2024-11-19T05:26:51,530 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0e56115b9d4bd5b1128cd15dcc8b22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:51,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/31aa467200a44ef190ea46f5b0b2e0dc, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:51,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/31aa467200a44ef190ea46f5b0b2e0dc is 175, key is test_row_0/A:col10/1731994010211/Put/seqid=0 2024-11-19T05:26:51,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742426_1602 (size=31971) 2024-11-19T05:26:51,546 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:51,546 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/5991c35889df43edb74a605a4c2eee10 is 50, key is test_row_0/C:col10/1731994009037/Put/seqid=0 2024-11-19T05:26:51,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742427_1603 (size=31255) 2024-11-19T05:26:51,552 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/31aa467200a44ef190ea46f5b0b2e0dc 2024-11-19T05:26:51,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742428_1604 (size=13017) 2024-11-19T05:26:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/8278dfb80471401a91c955c7d159590b is 50, key is test_row_0/B:col10/1731994010211/Put/seqid=0 2024-11-19T05:26:51,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742429_1605 (size=12301) 2024-11-19T05:26:51,941 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/0777b789c3b84d09a444c46232116317 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317 2024-11-19T05:26:51,945 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 0777b789c3b84d09a444c46232116317(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:51,945 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:51,945 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994011466; duration=0sec 2024-11-19T05:26:51,945 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:51,946 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:51,963 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/5991c35889df43edb74a605a4c2eee10 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5991c35889df43edb74a605a4c2eee10 2024-11-19T05:26:51,966 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 5991c35889df43edb74a605a4c2eee10(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:51,966 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:51,966 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994011467; duration=0sec 2024-11-19T05:26:51,967 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:51,967 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:51,972 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/8278dfb80471401a91c955c7d159590b 2024-11-19T05:26:51,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/b2f438caa7ea49749aaf90cca7da7a7d is 50, key is test_row_0/C:col10/1731994010211/Put/seqid=0 2024-11-19T05:26:51,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742430_1606 (size=12301) 2024-11-19T05:26:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. as already flushing 2024-11-19T05:26:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:52,381 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/b2f438caa7ea49749aaf90cca7da7a7d 2024-11-19T05:26:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/31aa467200a44ef190ea46f5b0b2e0dc as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc 2024-11-19T05:26:52,386 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc, entries=150, sequenceid=312, filesize=30.5 K 2024-11-19T05:26:52,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/8278dfb80471401a91c955c7d159590b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b 2024-11-19T05:26:52,390 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T05:26:52,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/b2f438caa7ea49749aaf90cca7da7a7d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d 2024-11-19T05:26:52,394 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d, entries=150, sequenceid=312, filesize=12.0 K 2024-11-19T05:26:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,402 INFO [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=26.84 KB/27480 for 3086702af0f4dab24249af64f43614f1 in 919ms, sequenceid=312, compaction requested=false 2024-11-19T05:26:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:52,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/f990f635ffd0:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-19T05:26:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-19T05:26:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,404 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:52,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-19T05:26:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-19T05:26:52,406 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1440 sec 2024-11-19T05:26:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1490 sec 2024-11-19T05:26:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119381f890a8bd945ad946700bf9da35b21_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994072434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994072436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994072437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994072441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994072441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742431_1607 (size=22618) 2024-11-19T05:26:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994072547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994072547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994072547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994072557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994072557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994072757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994072757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,761 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994072757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994072767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994072767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:52,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:52,850 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:52,852 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241119381f890a8bd945ad946700bf9da35b21_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119381f890a8bd945ad946700bf9da35b21_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:52,853 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f021f2eb604e4e1a9c8859eb9900061d, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:52,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f021f2eb604e4e1a9c8859eb9900061d is 175, key is test_row_0/A:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:52,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742432_1608 (size=66023) 2024-11-19T05:26:53,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994073067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994073067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994073067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994073077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994073077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,078 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f021f2eb604e4e1a9c8859eb9900061d 2024-11-19T05:26:53,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/63f3677b68f8431c97cdca955d147b74 is 50, key is test_row_0/B:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:53,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742433_1609 (size=12301) 2024-11-19T05:26:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35816 deadline: 1731994073577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35868 deadline: 1731994073577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35848 deadline: 1731994073577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35844 deadline: 1731994073587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38449 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35806 deadline: 1731994073587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 , the old value is region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-19T05:26:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., hostname=f990f635ffd0,38449,1731993838792, seqNum=5 because the exception is null or not the one we care about 2024-11-19T05:26:53,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/63f3677b68f8431c97cdca955d147b74 2024-11-19T05:26:53,671 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ad37ac1e52094c118a9608d40f696303 is 50, key is test_row_0/C:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:53,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742434_1610 (size=12301) 2024-11-19T05:26:53,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ad37ac1e52094c118a9608d40f696303 2024-11-19T05:26:53,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/f021f2eb604e4e1a9c8859eb9900061d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d 2024-11-19T05:26:53,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d, entries=350, sequenceid=327, filesize=64.5 K 2024-11-19T05:26:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/63f3677b68f8431c97cdca955d147b74 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74 2024-11-19T05:26:53,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T05:26:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/ad37ac1e52094c118a9608d40f696303 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303 2024-11-19T05:26:53,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303, entries=150, sequenceid=327, filesize=12.0 K 2024-11-19T05:26:53,693 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 3086702af0f4dab24249af64f43614f1 in 1288ms, sequenceid=327, compaction requested=true 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:A, priority=-2147483648, current under compaction store size is 1 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:B, priority=-2147483648, current under compaction store size is 2 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3086702af0f4dab24249af64f43614f1:C, priority=-2147483648, current under compaction store size is 3 2024-11-19T05:26:53,693 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129249 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/B is initiating minor compaction (all files) 2024-11-19T05:26:53,693 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/A is initiating minor compaction (all files) 2024-11-19T05:26:53,694 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/B in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:53,694 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/A in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:53,694 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/b710c51731ff4d88b28f578a68174f85, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.7 K 2024-11-19T05:26:53,694 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=126.2 K 2024-11-19T05:26:53,694 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:53,694 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. files: [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d] 2024-11-19T05:26:53,694 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0777b789c3b84d09a444c46232116317, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:53,694 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b710c51731ff4d88b28f578a68174f85, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:53,694 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 8278dfb80471401a91c955c7d159590b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731994010211 2024-11-19T05:26:53,694 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31aa467200a44ef190ea46f5b0b2e0dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731994010211 2024-11-19T05:26:53,695 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] compactions.Compactor(225): Compacting f021f2eb604e4e1a9c8859eb9900061d, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731994012368 2024-11-19T05:26:53,695 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 63f3677b68f8431c97cdca955d147b74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731994012368 2024-11-19T05:26:53,700 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:53,701 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411194d4d8309ffce468b9d5347bc6d4c0a3e_3086702af0f4dab24249af64f43614f1 store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:53,702 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#B#compaction#513 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:53,702 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/19a6a53d66b64321a345b06fff385fc8 is 50, key is test_row_0/B:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:53,704 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411194d4d8309ffce468b9d5347bc6d4c0a3e_3086702af0f4dab24249af64f43614f1, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:53,704 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411194d4d8309ffce468b9d5347bc6d4c0a3e_3086702af0f4dab24249af64f43614f1 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:53,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742435_1611 (size=13119) 2024-11-19T05:26:53,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742436_1612 (size=4469) 2024-11-19T05:26:53,915 INFO [Thread-2313 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2313. 2024-11-19T05:26:53,915 DEBUG [Thread-2313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:53,915 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,915 DEBUG [Thread-2313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:53,915 INFO [Thread-2319 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2319. 2024-11-19T05:26:53,915 DEBUG [Thread-2319 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:53,915 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,915 DEBUG [Thread-2319 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,915 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:53,916 INFO [Thread-2315 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2315. 2024-11-19T05:26:53,916 DEBUG [Thread-2315 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:53,916 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,916 DEBUG [Thread-2315 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,916 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:53,916 INFO [Thread-2321 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2321. 2024-11-19T05:26:53,916 DEBUG [Thread-2321 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:53,916 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,916 DEBUG [Thread-2321 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:53,918 INFO [Thread-2317 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2317. 2024-11-19T05:26:53,918 DEBUG [Thread-2317 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:53,918 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,918 DEBUG [Thread-2317 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:53,918 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,109 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/19a6a53d66b64321a345b06fff385fc8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/19a6a53d66b64321a345b06fff385fc8 2024-11-19T05:26:54,113 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/B of 3086702af0f4dab24249af64f43614f1 into 19a6a53d66b64321a345b06fff385fc8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:54,113 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:54,113 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/B, priority=13, startTime=1731994013693; duration=0sec 2024-11-19T05:26:54,113 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-19T05:26:54,113 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:B 2024-11-19T05:26:54,113 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-19T05:26:54,114 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-19T05:26:54,114 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1541): 3086702af0f4dab24249af64f43614f1/C is initiating minor compaction (all files) 2024-11-19T05:26:54,114 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3086702af0f4dab24249af64f43614f1/C in TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:54,114 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5991c35889df43edb74a605a4c2eee10, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303] into tmpdir=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp, totalSize=36.7 K 2024-11-19T05:26:54,114 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting 5991c35889df43edb74a605a4c2eee10, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731994009037 2024-11-19T05:26:54,114 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting b2f438caa7ea49749aaf90cca7da7a7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731994010211 2024-11-19T05:26:54,114 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] compactions.Compactor(225): Compacting ad37ac1e52094c118a9608d40f696303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731994012368 2024-11-19T05:26:54,118 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#C#compaction#514 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:54,119 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/3d635d43843046c289b59cf9f47f5525 is 50, key is test_row_0/C:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:54,120 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3086702af0f4dab24249af64f43614f1#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-19T05:26:54,120 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/3306af7c280344b4aaf602a12f5de5ed is 175, key is test_row_0/A:col10/1731994012368/Put/seqid=0 2024-11-19T05:26:54,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742437_1613 (size=13119) 2024-11-19T05:26:54,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742438_1614 (size=32073) 2024-11-19T05:26:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-19T05:26:54,417 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:54,525 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/3d635d43843046c289b59cf9f47f5525 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/3d635d43843046c289b59cf9f47f5525 2024-11-19T05:26:54,526 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/3306af7c280344b4aaf602a12f5de5ed as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/3306af7c280344b4aaf602a12f5de5ed 2024-11-19T05:26:54,529 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/A of 3086702af0f4dab24249af64f43614f1 into 3306af7c280344b4aaf602a12f5de5ed(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:54,529 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3086702af0f4dab24249af64f43614f1/C of 3086702af0f4dab24249af64f43614f1 into 3d635d43843046c289b59cf9f47f5525(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:54,529 INFO [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/A, priority=13, startTime=1731994013693; duration=0sec 2024-11-19T05:26:54,529 INFO [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., storeName=3086702af0f4dab24249af64f43614f1/C, priority=13, startTime=1731994013693; duration=0sec 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:A 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-19T05:26:54,529 DEBUG [RS:0;f990f635ffd0:38449-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3086702af0f4dab24249af64f43614f1:C 2024-11-19T05:26:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38449 {}] regionserver.HRegion(8855): Flush requested on 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:54,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-19T05:26:54,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-19T05:26:54,589 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-19T05:26:54,589 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:54,589 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-19T05:26:54,589 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-19T05:26:54,589 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:54,589 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:54,589 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908946ea39b674646a5297006b05899f3_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_0/A:col10/1731994014588/Put/seqid=0 2024-11-19T05:26:54,598 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-19T05:26:54,598 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:54,598 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,598 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,598 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742439_1615 (size=12454) 2024-11-19T05:26:54,608 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-19T05:26:54,608 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-19T05:26:54,608 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,608 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3606 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3511 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3559 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3585 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3510 2024-11-19T05:26:54,609 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-19T05:26:54,609 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:26:54,609 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:54,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,609 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:54,609 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:54,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-19T05:26:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:54,612 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731994014612"}]},"ts":"1731994014612"} 2024-11-19T05:26:54,613 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-19T05:26:54,613 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-19T05:26:54,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-19T05:26:54,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, UNASSIGN}] 2024-11-19T05:26:54,616 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, UNASSIGN 2024-11-19T05:26:54,616 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=CLOSING, regionLocation=f990f635ffd0,38449,1731993838792 2024-11-19T05:26:54,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, UNASSIGN because future has completed 2024-11-19T05:26:54,618 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-19T05:26:54,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792}] 2024-11-19T05:26:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:54,770 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:54,770 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-19T05:26:54,770 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing 3086702af0f4dab24249af64f43614f1, disabling compactions & flushes 2024-11-19T05:26:54,770 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:54,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:55,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111908946ea39b674646a5297006b05899f3_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908946ea39b674646a5297006b05899f3_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:55,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:55,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 is 175, key is test_row_0/A:col10/1731994014588/Put/seqid=0 2024-11-19T05:26:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742440_1616 (size=31255) 2024-11-19T05:26:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:55,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 2024-11-19T05:26:55,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/be6dca4a5e134488a7a1ac21ba87d1c8 is 50, key is test_row_0/B:col10/1731994014588/Put/seqid=0 2024-11-19T05:26:55,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742441_1617 (size=12301) 2024-11-19T05:26:55,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:55,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/be6dca4a5e134488a7a1ac21ba87d1c8 2024-11-19T05:26:55,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2a9ace0efd734f0da3832a36c69ec1fa is 50, key is test_row_0/C:col10/1731994014588/Put/seqid=0 2024-11-19T05:26:55,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742442_1618 (size=12301) 2024-11-19T05:26:56,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2a9ace0efd734f0da3832a36c69ec1fa 2024-11-19T05:26:56,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 2024-11-19T05:26:56,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0, entries=150, sequenceid=358, filesize=30.5 K 2024-11-19T05:26:56,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/be6dca4a5e134488a7a1ac21ba87d1c8 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/be6dca4a5e134488a7a1ac21ba87d1c8 2024-11-19T05:26:56,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/be6dca4a5e134488a7a1ac21ba87d1c8, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T05:26:56,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2a9ace0efd734f0da3832a36c69ec1fa as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2a9ace0efd734f0da3832a36c69ec1fa 2024-11-19T05:26:56,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2a9ace0efd734f0da3832a36c69ec1fa, entries=150, sequenceid=358, filesize=12.0 K 2024-11-19T05:26:56,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=13.42 KB/13740 for 3086702af0f4dab24249af64f43614f1 in 1662ms, sequenceid=358, compaction requested=false 2024-11-19T05:26:56,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3086702af0f4dab24249af64f43614f1: 2024-11-19T05:26:56,251 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:56,251 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:56,251 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. after waiting 0 ms 2024-11-19T05:26:56,251 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:56,251 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing 3086702af0f4dab24249af64f43614f1 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-19T05:26:56,256 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111966251b5f041a4bbaaabc976601873bc5_3086702af0f4dab24249af64f43614f1 is 50, key is test_row_1/A:col10/1731994014597/Put/seqid=0 2024-11-19T05:26:56,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742443_1619 (size=9914) 2024-11-19T05:26:56,661 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T05:26:56,669 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111966251b5f041a4bbaaabc976601873bc5_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111966251b5f041a4bbaaabc976601873bc5_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:56,670 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/63f2c939da6f4512b18ba7ef34699359, store: [table=TestAcidGuarantees family=A region=3086702af0f4dab24249af64f43614f1] 2024-11-19T05:26:56,671 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/63f2c939da6f4512b18ba7ef34699359 is 175, key is test_row_1/A:col10/1731994014597/Put/seqid=0 2024-11-19T05:26:56,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742444_1620 (size=22561) 2024-11-19T05:26:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:57,078 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/63f2c939da6f4512b18ba7ef34699359 2024-11-19T05:26:57,090 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/d7661766bb9446e5a9ec8d926937a5ed is 50, key is test_row_1/B:col10/1731994014597/Put/seqid=0 2024-11-19T05:26:57,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742445_1621 (size=9857) 2024-11-19T05:26:57,116 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-19T05:26:57,495 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/d7661766bb9446e5a9ec8d926937a5ed 2024-11-19T05:26:57,500 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2fd98a9f62b54fb58274767dae3e6fad is 50, key is test_row_1/C:col10/1731994014597/Put/seqid=0 2024-11-19T05:26:57,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742446_1622 (size=9857) 2024-11-19T05:26:57,905 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2fd98a9f62b54fb58274767dae3e6fad 2024-11-19T05:26:57,914 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/A/63f2c939da6f4512b18ba7ef34699359 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/63f2c939da6f4512b18ba7ef34699359 2024-11-19T05:26:57,918 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/63f2c939da6f4512b18ba7ef34699359, entries=100, sequenceid=363, filesize=22.0 K 2024-11-19T05:26:57,919 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/B/d7661766bb9446e5a9ec8d926937a5ed as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/d7661766bb9446e5a9ec8d926937a5ed 2024-11-19T05:26:57,922 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/d7661766bb9446e5a9ec8d926937a5ed, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T05:26:57,923 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/.tmp/C/2fd98a9f62b54fb58274767dae3e6fad as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2fd98a9f62b54fb58274767dae3e6fad 2024-11-19T05:26:57,927 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2fd98a9f62b54fb58274767dae3e6fad, entries=100, sequenceid=363, filesize=9.6 K 2024-11-19T05:26:57,928 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3086702af0f4dab24249af64f43614f1 in 1677ms, sequenceid=363, compaction requested=true 2024-11-19T05:26:57,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d] to archive 2024-11-19T05:26:57,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:57,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/19dbeb9785284297a418859430ed8c7b 2024-11-19T05:26:57,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a6aa94388a494bc59d7ea7a2e4a541bd 2024-11-19T05:26:57,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1049af3858814db5a9dc89ab2a51f86c 2024-11-19T05:26:57,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/a9f3d933d2834057864c0f6cbaab1a7b 2024-11-19T05:26:57,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f6e045d8015941a6a99de4d2977027ef 2024-11-19T05:26:57,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/20fcb2b59463435dacbc3e5b8d5f5596 2024-11-19T05:26:57,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/8674b72a08b246ccb488d7dddad41c4f 2024-11-19T05:26:57,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c398e0ca5f6a42b98dbe43620953f0e2 2024-11-19T05:26:57,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/648a589f8fb042b4aa87cc58c1432bc7 2024-11-19T05:26:57,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/828821a9f9c548bdbe8d631c45750abd 2024-11-19T05:26:57,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/15b578acc63440c98f42e7f7e2d1eef3 2024-11-19T05:26:57,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/871880031907402a8764c2b8807cd451 2024-11-19T05:26:57,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/73c88cb17c384c52a4b1b130ba2f24f6 2024-11-19T05:26:57,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/93e4072e24f1400cb8041aaf5fb86075 2024-11-19T05:26:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/af86ac38ce9045909472e2e9b678f949 2024-11-19T05:26:57,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/77e710f196c24ac188d0517e6099bc0a 2024-11-19T05:26:57,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/ad8e173175e445c2bdd9438295777148 2024-11-19T05:26:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/5f0ad32b02204bb1acec604562e7d38f 2024-11-19T05:26:57,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/36642fff141f495eb8c976752380c582 2024-11-19T05:26:57,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/1f893b1f43474e69ba087117b8f624b1 2024-11-19T05:26:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/234799519b0549c59596173960d42721 2024-11-19T05:26:57,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/0777b789c3b84d09a444c46232116317 2024-11-19T05:26:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/31aa467200a44ef190ea46f5b0b2e0dc 2024-11-19T05:26:57,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/f021f2eb604e4e1a9c8859eb9900061d 2024-11-19T05:26:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/f295b2eae5fb42859f1e5094aa1aff5d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/14b0946fb3834830b0d3e603bc87dfbf, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9d5c0e79061f463f8959e04417120f1b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/87c4a1f6ec9b4d4faa2b19211a456f68, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2406ed06eacc4a6694892dc4105b257c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/692af004793e4fc7a9e6c3549212926a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/b710c51731ff4d88b28f578a68174f85, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74] to archive 2024-11-19T05:26:57,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:57,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/5f6a2188299744cf938208c0bd9af48d 2024-11-19T05:26:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/c3ecd2cc94e446b4b9ecd3d1b9a8656c 2024-11-19T05:26:57,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/f295b2eae5fb42859f1e5094aa1aff5d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/f295b2eae5fb42859f1e5094aa1aff5d 2024-11-19T05:26:57,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/1acf6bef3d97489e9ff8fede7bc4bbc4 2024-11-19T05:26:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/0246218150cb455fb7ba0cfbcd94d8d1 2024-11-19T05:26:57,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/14b0946fb3834830b0d3e603bc87dfbf to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/14b0946fb3834830b0d3e603bc87dfbf 2024-11-19T05:26:57,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/4ce7df5b94e644a2a9545382cd63367e 2024-11-19T05:26:57,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/05f21bce907c48bda31e4c5d5c557106 2024-11-19T05:26:57,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9d5c0e79061f463f8959e04417120f1b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9d5c0e79061f463f8959e04417120f1b 2024-11-19T05:26:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/6881dbc64df24fd086adcbed4135b27b 2024-11-19T05:26:57,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/7af9b872939148059de55d9e7f8ce8fd 2024-11-19T05:26:57,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/87c4a1f6ec9b4d4faa2b19211a456f68 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/87c4a1f6ec9b4d4faa2b19211a456f68 2024-11-19T05:26:57,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2c2c5b6d741741ada6808c9720de1e81 2024-11-19T05:26:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2cc7a1ef6213420da01c382305831007 2024-11-19T05:26:57,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2406ed06eacc4a6694892dc4105b257c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/2406ed06eacc4a6694892dc4105b257c 2024-11-19T05:26:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/ed69f237ef064c279de7276f8241a84e 2024-11-19T05:26:57,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/16d3f4d79ec44f59b28d4721b7b9667d 2024-11-19T05:26:57,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/692af004793e4fc7a9e6c3549212926a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/692af004793e4fc7a9e6c3549212926a 2024-11-19T05:26:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/9086bab4b67048ccad44139667e2de5a 2024-11-19T05:26:57,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/679c7902714a4281a233cd23e054cf5f 2024-11-19T05:26:57,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/b710c51731ff4d88b28f578a68174f85 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/b710c51731ff4d88b28f578a68174f85 2024-11-19T05:26:57,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/a05e4da19f9d423da2929be6a0aa829c 2024-11-19T05:26:57,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/8278dfb80471401a91c955c7d159590b 2024-11-19T05:26:57,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/63f3677b68f8431c97cdca955d147b74 2024-11-19T05:26:57,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7d158bfa66b74d7884f2f8c313d2a85a, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5432d08bc87d4f30bbed57d66e4ec08e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/adbb9af7831941299da671e0a6bfdbdd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/51081e6db14b4d5cb0c01d725884f21e, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7489fc013d014db1b9c27965ef90ec15, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a05dcfb872d54069bce59dac5037a5bd, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5991c35889df43edb74a605a4c2eee10, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303] to archive 2024-11-19T05:26:57,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-19T05:26:57,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ede76d269d12462eaeeb2e814af19d68 2024-11-19T05:26:57,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/13d6e25fe2b842dcbc44eba1b03deae3 2024-11-19T05:26:57,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7d158bfa66b74d7884f2f8c313d2a85a to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7d158bfa66b74d7884f2f8c313d2a85a 2024-11-19T05:26:57,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/4cf613c883784147a63f9ab530aacae6 2024-11-19T05:26:57,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/75b516cf07664ac5b1c635109a1ee238 2024-11-19T05:26:57,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5432d08bc87d4f30bbed57d66e4ec08e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5432d08bc87d4f30bbed57d66e4ec08e 2024-11-19T05:26:57,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ef72464a19c48de9adafc263d066afa 2024-11-19T05:26:57,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7ac4a6eaf0be4572b74131149389cd62 2024-11-19T05:26:57,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/adbb9af7831941299da671e0a6bfdbdd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/adbb9af7831941299da671e0a6bfdbdd 2024-11-19T05:26:57,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fd52060022a74f19978fa1d517f4c1e5 2024-11-19T05:26:57,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f4825d637c1d461caa59e97fcc87f245 2024-11-19T05:26:57,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/51081e6db14b4d5cb0c01d725884f21e to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/51081e6db14b4d5cb0c01d725884f21e 2024-11-19T05:26:57,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/984d06ef215f44c5a4b8be39e334acbc 2024-11-19T05:26:57,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a4655607388341e59cc1be712a0ed982 2024-11-19T05:26:57,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7489fc013d014db1b9c27965ef90ec15 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/7489fc013d014db1b9c27965ef90ec15 2024-11-19T05:26:57,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/d6cdaaa4ec69484a95699dcbb18569b1 2024-11-19T05:26:57,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/f31a501f8990425ba9dbafc5ce05a076 2024-11-19T05:26:57,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a05dcfb872d54069bce59dac5037a5bd to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/a05dcfb872d54069bce59dac5037a5bd 2024-11-19T05:26:57,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fb7f560c65e447abb8a3ce8c373cb9e4 2024-11-19T05:26:57,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/cb59262991814ccab85c35cb0f441049 2024-11-19T05:26:58,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5991c35889df43edb74a605a4c2eee10 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/5991c35889df43edb74a605a4c2eee10 2024-11-19T05:26:58,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/fe0e56115b9d4bd5b1128cd15dcc8b22 2024-11-19T05:26:58,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/b2f438caa7ea49749aaf90cca7da7a7d 2024-11-19T05:26:58,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/ad37ac1e52094c118a9608d40f696303 2024-11-19T05:26:58,009 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits/366.seqid, newMaxSeqId=366, maxSeqId=4 2024-11-19T05:26:58,010 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. 2024-11-19T05:26:58,010 DEBUG [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for 3086702af0f4dab24249af64f43614f1: Waiting for close lock at 1731994014770Running coprocessor pre-close hooks at 1731994014770Disabling compacts and flushes for region at 1731994014770Disabling writes for close at 1731994016251 (+1481 ms)Obtaining lock to block concurrent updates at 1731994016251Preparing flush snapshotting stores in 3086702af0f4dab24249af64f43614f1 at 1731994016251Finished memstore snapshotting TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731994016251Flushing stores of TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1. at 1731994016252 (+1 ms)Flushing 3086702af0f4dab24249af64f43614f1/A: creating writer at 1731994016252Flushing mob file 3086702af0f4dab24249af64f43614f1/A: appending metadata at 1731994016256 (+4 ms)Flushing mob file 3086702af0f4dab24249af64f43614f1/A: closing flushed file at 1731994016256Flushing 3086702af0f4dab24249af64f43614f1/A: appending metadata at 1731994016670 (+414 ms)Flushing 3086702af0f4dab24249af64f43614f1/A: closing flushed file at 1731994016670Flushing 3086702af0f4dab24249af64f43614f1/B: creating writer at 1731994017086 (+416 ms)Flushing 3086702af0f4dab24249af64f43614f1/B: appending metadata at 1731994017089 (+3 ms)Flushing 3086702af0f4dab24249af64f43614f1/B: closing flushed file at 1731994017089Flushing 3086702af0f4dab24249af64f43614f1/C: creating writer at 1731994017498 (+409 ms)Flushing 3086702af0f4dab24249af64f43614f1/C: appending metadata at 1731994017499 (+1 ms)Flushing 3086702af0f4dab24249af64f43614f1/C: closing flushed file at 1731994017500 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c95e6f2: reopening flushed file at 1731994017914 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43215390: reopening flushed file at 1731994017918 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c11aa2b: reopening flushed file at 1731994017923 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3086702af0f4dab24249af64f43614f1 in 1677ms, sequenceid=363, compaction requested=true at 1731994017928 (+5 ms)Writing region close event to WAL at 1731994018006 (+78 ms)Running coprocessor post-close hooks at 1731994018010 (+4 ms)Closed at 1731994018010 2024-11-19T05:26:58,013 INFO [RS_CLOSE_REGION-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed 3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,013 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=3086702af0f4dab24249af64f43614f1, regionState=CLOSED 2024-11-19T05:26:58,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 because future has completed 2024-11-19T05:26:58,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-19T05:26:58,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure 3086702af0f4dab24249af64f43614f1, server=f990f635ffd0,38449,1731993838792 in 3.3980 sec 2024-11-19T05:26:58,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-19T05:26:58,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3086702af0f4dab24249af64f43614f1, UNASSIGN in 3.4040 sec 2024-11-19T05:26:58,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-19T05:26:58,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 3.4090 sec 2024-11-19T05:26:58,028 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731994018028"}]},"ts":"1731994018028"} 2024-11-19T05:26:58,030 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-19T05:26:58,031 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-19T05:26:58,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.4210 sec 2024-11-19T05:26:58,366 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-19T05:26:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-19T05:26:58,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:58,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-19T05:26:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,780 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,780 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T05:26:58,782 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,784 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C, FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits] 2024-11-19T05:26:58,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/3306af7c280344b4aaf602a12f5de5ed to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/3306af7c280344b4aaf602a12f5de5ed 2024-11-19T05:26:58,787 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/63f2c939da6f4512b18ba7ef34699359 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/63f2c939da6f4512b18ba7ef34699359 2024-11-19T05:26:58,787 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/A/c08d74fbb2dd4711bfc6f5cc4a62dcd0 2024-11-19T05:26:58,789 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/19a6a53d66b64321a345b06fff385fc8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/19a6a53d66b64321a345b06fff385fc8 2024-11-19T05:26:58,790 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/be6dca4a5e134488a7a1ac21ba87d1c8 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/be6dca4a5e134488a7a1ac21ba87d1c8 2024-11-19T05:26:58,791 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/d7661766bb9446e5a9ec8d926937a5ed to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/B/d7661766bb9446e5a9ec8d926937a5ed 2024-11-19T05:26:58,793 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2a9ace0efd734f0da3832a36c69ec1fa to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2a9ace0efd734f0da3832a36c69ec1fa 2024-11-19T05:26:58,794 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2fd98a9f62b54fb58274767dae3e6fad to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/2fd98a9f62b54fb58274767dae3e6fad 2024-11-19T05:26:58,795 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/3d635d43843046c289b59cf9f47f5525 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/C/3d635d43843046c289b59cf9f47f5525 2024-11-19T05:26:58,797 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits/366.seqid to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1/recovered.edits/366.seqid 2024-11-19T05:26:58,797 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/default/TestAcidGuarantees/3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,797 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-19T05:26:58,798 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:26:58,798 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-19T05:26:58,801 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908946ea39b674646a5297006b05899f3_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111908946ea39b674646a5297006b05899f3_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,802 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912db1cd5e6414ab1be20025033e8c7ff_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111912db1cd5e6414ab1be20025033e8c7ff_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,803 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191f433fe8b3514c5d8f51098d2a378e3f_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411191f433fe8b3514c5d8f51098d2a378e3f_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,804 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238686d614564325a6c0e3844784b510_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119238686d614564325a6c0e3844784b510_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,806 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192feed167265a432480a498d94bfd177b_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411192feed167265a432480a498d94bfd177b_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,807 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119353b433252d64cfbba324db6cc24b8cd_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119353b433252d64cfbba324db6cc24b8cd_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,808 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119381f890a8bd945ad946700bf9da35b21_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119381f890a8bd945ad946700bf9da35b21_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,809 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f602c397fef44ec839a69cd1f678347_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411193f602c397fef44ec839a69cd1f678347_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,810 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119405e04f6abcf4e92be39d9dd9a1f40fc_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119405e04f6abcf4e92be39d9dd9a1f40fc_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,811 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c34cd8680814c93810a003f6ad1759e_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411194c34cd8680814c93810a003f6ad1759e_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,812 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195bee869936754a13b0eb7bb798206edd_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411195bee869936754a13b0eb7bb798206edd_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,814 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111966251b5f041a4bbaaabc976601873bc5_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111966251b5f041a4bbaaabc976601873bc5_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,815 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c9a72d7804f44f5ab564cf9de2ba626_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411196c9a72d7804f44f5ab564cf9de2ba626_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198508335a1c02464e92b51da2c6a7d3ac_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411198508335a1c02464e92b51da2c6a7d3ac_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,816 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a36f033514634a67b5321c5fdc2f7423_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119a36f033514634a67b5321c5fdc2f7423_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,817 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acc07389bcb743b490cfe9fa5baeebd3_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119acc07389bcb743b490cfe9fa5baeebd3_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,818 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9ec519df38f4e05893cf053b00d0403_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119b9ec519df38f4e05893cf053b00d0403_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,819 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db5a05cadc334fc2b9918551ff0b35e2_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119db5a05cadc334fc2b9918551ff0b35e2_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed88135217a64ec89926563ae64b824c_3086702af0f4dab24249af64f43614f1 to hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241119ed88135217a64ec89926563ae64b824c_3086702af0f4dab24249af64f43614f1 2024-11-19T05:26:58,820 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-19T05:26:58,822 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,824 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-19T05:26:58,826 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-19T05:26:58,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,827 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-19T05:26:58,827 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731994018827"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:58,829 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-19T05:26:58,829 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3086702af0f4dab24249af64f43614f1, NAME => 'TestAcidGuarantees,,1731993990805.3086702af0f4dab24249af64f43614f1.', STARTKEY => '', ENDKEY => ''}] 2024-11-19T05:26:58,829 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-19T05:26:58,829 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731994018829"}]},"ts":"9223372036854775807"} 2024-11-19T05:26:58,830 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-19T05:26:58,831 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-19T05:26:58,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-19T05:26:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34055 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-19T05:26:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-19T05:26:58,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-19T05:26:58,895 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 240), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=501 (was 554), ProcessCount=11 (was 11), AvailableMemoryMB=11625 (was 11640) 2024-11-19T05:26:58,895 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T05:26:58,896 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:26:58,896 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:58,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:58,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:58,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T05:26:58,896 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T05:26:58,896 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=694036216, stopped=false 2024-11-19T05:26:58,897 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f990f635ffd0,34055,1731993838016 2024-11-19T05:26:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T05:26:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T05:26:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:26:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:26:58,953 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T05:26:58,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T05:26:58,953 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T05:26:58,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T05:26:58,954 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:58,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:58,954 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f990f635ffd0,38449,1731993838792' ***** 2024-11-19T05:26:58,954 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T05:26:58,954 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T05:26:58,954 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T05:26:58,954 INFO [RS:0;f990f635ffd0:38449 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(959): stopping server f990f635ffd0,38449,1731993838792 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f990f635ffd0:38449. 2024-11-19T05:26:58,955 DEBUG [RS:0;f990f635ffd0:38449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T05:26:58,955 DEBUG [RS:0;f990f635ffd0:38449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T05:26:58,955 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T05:26:58,956 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T05:26:58,956 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T05:26:58,956 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:26:58,956 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T05:26:58,956 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T05:26:58,956 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T05:26:58,956 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T05:26:58,956 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T05:26:58,956 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-19T05:26:58,957 INFO [regionserver/f990f635ffd0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T05:26:58,981 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/info/a6a170bb55124f5a9014a3a41a2e6e56 is 95, key is TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824./info:/1731993875635/DeleteFamily/seqid=0 2024-11-19T05:26:58,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742447_1623 (size=6514) 2024-11-19T05:26:58,984 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/info/a6a170bb55124f5a9014a3a41a2e6e56 2024-11-19T05:26:59,006 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/ns/09406a038de3418b805519b4d953870b is 93, key is TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824./ns:/1731993875588/DeleteFamily/seqid=0 2024-11-19T05:26:59,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742448_1624 (size=5906) 2024-11-19T05:26:59,156 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:26:59,356 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:26:59,410 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/ns/09406a038de3418b805519b4d953870b 2024-11-19T05:26:59,431 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/rep_barrier/e5b4bf7503844295a9c513342330c688 is 102, key is TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824./rep_barrier:/1731993875588/DeleteFamily/seqid=0 2024-11-19T05:26:59,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742449_1625 (size=6025) 2024-11-19T05:26:59,552 INFO [regionserver/f990f635ffd0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-19T05:26:59,553 INFO [regionserver/f990f635ffd0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-19T05:26:59,557 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:26:59,757 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:26:59,835 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/rep_barrier/e5b4bf7503844295a9c513342330c688 2024-11-19T05:26:59,850 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/table/a6c1e243cf60479189b9a6d29e870e82 is 96, key is TestAcidGuarantees,,1731993842700.911bdd839d983af04003ffd0fc873824./table:/1731993875588/DeleteFamily/seqid=0 2024-11-19T05:26:59,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742450_1626 (size=5936) 2024-11-19T05:26:59,957 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:27:00,157 DEBUG [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T05:27:00,254 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/table/a6c1e243cf60479189b9a6d29e870e82 2024-11-19T05:27:00,257 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/info/a6a170bb55124f5a9014a3a41a2e6e56 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/info/a6a170bb55124f5a9014a3a41a2e6e56 2024-11-19T05:27:00,259 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/info/a6a170bb55124f5a9014a3a41a2e6e56, entries=12, sequenceid=90, filesize=6.4 K 2024-11-19T05:27:00,259 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/ns/09406a038de3418b805519b4d953870b as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/ns/09406a038de3418b805519b4d953870b 2024-11-19T05:27:00,261 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/ns/09406a038de3418b805519b4d953870b, entries=8, sequenceid=90, filesize=5.8 K 2024-11-19T05:27:00,262 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/rep_barrier/e5b4bf7503844295a9c513342330c688 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/rep_barrier/e5b4bf7503844295a9c513342330c688 2024-11-19T05:27:00,264 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/rep_barrier/e5b4bf7503844295a9c513342330c688, entries=6, sequenceid=90, filesize=5.9 K 2024-11-19T05:27:00,264 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/.tmp/table/a6c1e243cf60479189b9a6d29e870e82 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/table/a6c1e243cf60479189b9a6d29e870e82 2024-11-19T05:27:00,266 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/table/a6c1e243cf60479189b9a6d29e870e82, entries=7, sequenceid=90, filesize=5.8 K 2024-11-19T05:27:00,267 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=90, compaction requested=false 2024-11-19T05:27:00,270 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-19T05:27:00,270 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T05:27:00,270 INFO [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T05:27:00,270 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731994018956Running coprocessor pre-close hooks at 1731994018956Disabling compacts and flushes for region at 1731994018956Disabling writes for close at 1731994018956Obtaining lock to block concurrent updates at 1731994018956Preparing flush snapshotting stores in 1588230740 at 1731994018956Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731994018957 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731994018957Flushing 1588230740/info: creating writer at 1731994018957Flushing 1588230740/info: appending metadata at 1731994018980 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731994018980Flushing 1588230740/ns: creating writer at 1731994018987 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731994019006 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731994019006Flushing 1588230740/rep_barrier: creating writer at 1731994019413 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731994019431 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731994019431Flushing 1588230740/table: creating writer at 1731994019838 (+407 ms)Flushing 1588230740/table: appending metadata at 1731994019850 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731994019850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77188197: reopening flushed file at 1731994020256 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@775b804b: reopening flushed file at 1731994020259 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5b852a: reopening flushed file at 1731994020261 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba2ffc0: reopening flushed file at 1731994020264 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1311ms, sequenceid=90, compaction requested=false at 1731994020267 (+3 ms)Writing region close event to WAL at 1731994020268 (+1 ms)Running coprocessor post-close hooks at 1731994020270 (+2 ms)Closed at 1731994020270 2024-11-19T05:27:00,270 DEBUG [RS_CLOSE_META-regionserver/f990f635ffd0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T05:27:00,357 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(976): stopping server f990f635ffd0,38449,1731993838792; all regions closed. 2024-11-19T05:27:00,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741834_1010 (size=25344) 2024-11-19T05:27:00,363 DEBUG [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/oldWALs 2024-11-19T05:27:00,363 INFO [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f990f635ffd0%2C38449%2C1731993838792.meta:.meta(num 1731993842183) 2024-11-19T05:27:00,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741833_1009 (size=13966349) 2024-11-19T05:27:00,366 DEBUG [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/oldWALs 2024-11-19T05:27:00,366 INFO [RS:0;f990f635ffd0:38449 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f990f635ffd0%2C38449%2C1731993838792:(num 1731993841732) 2024-11-19T05:27:00,366 DEBUG [RS:0;f990f635ffd0:38449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T05:27:00,366 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T05:27:00,366 INFO [RS:0;f990f635ffd0:38449 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T05:27:00,367 INFO [RS:0;f990f635ffd0:38449 {}] hbase.ChoreService(370): Chore service for: regionserver/f990f635ffd0:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T05:27:00,367 INFO [RS:0;f990f635ffd0:38449 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T05:27:00,367 INFO [regionserver/f990f635ffd0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T05:27:00,367 INFO [RS:0;f990f635ffd0:38449 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38449 2024-11-19T05:27:00,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f990f635ffd0,38449,1731993838792 2024-11-19T05:27:00,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T05:27:00,426 INFO [RS:0;f990f635ffd0:38449 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T05:27:00,437 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f990f635ffd0,38449,1731993838792] 2024-11-19T05:27:00,447 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f990f635ffd0,38449,1731993838792 already deleted, retry=false 2024-11-19T05:27:00,447 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f990f635ffd0,38449,1731993838792 expired; onlineServers=0 2024-11-19T05:27:00,447 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f990f635ffd0,34055,1731993838016' ***** 2024-11-19T05:27:00,447 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T05:27:00,448 INFO [M:0;f990f635ffd0:34055 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T05:27:00,448 INFO [M:0;f990f635ffd0:34055 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T05:27:00,448 DEBUG [M:0;f990f635ffd0:34055 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T05:27:00,448 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T05:27:00,448 DEBUG [M:0;f990f635ffd0:34055 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T05:27:00,448 DEBUG [master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.large.0-1731993841345 {}] cleaner.HFileCleaner(306): Exit Thread[master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.large.0-1731993841345,5,FailOnTimeoutGroup] 2024-11-19T05:27:00,448 DEBUG [master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.small.0-1731993841348 {}] cleaner.HFileCleaner(306): Exit Thread[master/f990f635ffd0:0:becomeActiveMaster-HFileCleaner.small.0-1731993841348,5,FailOnTimeoutGroup] 2024-11-19T05:27:00,448 INFO [M:0;f990f635ffd0:34055 {}] hbase.ChoreService(370): Chore service for: master/f990f635ffd0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T05:27:00,448 INFO [M:0;f990f635ffd0:34055 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T05:27:00,448 DEBUG [M:0;f990f635ffd0:34055 {}] master.HMaster(1795): Stopping service threads 2024-11-19T05:27:00,448 INFO [M:0;f990f635ffd0:34055 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T05:27:00,449 INFO [M:0;f990f635ffd0:34055 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T05:27:00,449 ERROR [M:0;f990f635ffd0:34055 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-19T05:27:00,449 INFO [M:0;f990f635ffd0:34055 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T05:27:00,449 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T05:27:00,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T05:27:00,458 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T05:27:00,458 DEBUG [M:0;f990f635ffd0:34055 {}] zookeeper.ZKUtil(347): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T05:27:00,458 WARN [M:0;f990f635ffd0:34055 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T05:27:00,458 INFO [M:0;f990f635ffd0:34055 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/.lastflushedseqids 2024-11-19T05:27:00,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742451_1627 (size=72) 2024-11-19T05:27:00,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T05:27:00,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38449-0x101518c9a1c0001, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T05:27:00,537 INFO [RS:0;f990f635ffd0:38449 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T05:27:00,538 INFO [RS:0;f990f635ffd0:38449 {}] regionserver.HRegionServer(1031): Exiting; stopping=f990f635ffd0,38449,1731993838792; zookeeper connection closed. 2024-11-19T05:27:00,538 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5186ddd1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5186ddd1 2024-11-19T05:27:00,538 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-19T05:27:00,869 INFO [M:0;f990f635ffd0:34055 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T05:27:00,869 INFO [M:0;f990f635ffd0:34055 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T05:27:00,869 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T05:27:00,869 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:27:00,869 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:27:00,869 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T05:27:00,869 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:27:00,869 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=732.64 KB heapSize=898.42 KB 2024-11-19T05:27:00,890 DEBUG [M:0;f990f635ffd0:34055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a79d3f3184de4abfae15c493b656f7ae is 82, key is hbase:meta,,1/info:regioninfo/1731993842311/Put/seqid=0 2024-11-19T05:27:00,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742452_1628 (size=5672) 2024-11-19T05:27:01,294 INFO [M:0;f990f635ffd0:34055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a79d3f3184de4abfae15c493b656f7ae 2024-11-19T05:27:01,324 DEBUG [M:0;f990f635ffd0:34055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628f8d63517f4645b8dc7d7baf4a2034 is 2276, key is \x00\x00\x00\x00\x00\x00\x00V/proc:d/1731993940920/Put/seqid=0 2024-11-19T05:27:01,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742453_1629 (size=42802) 2024-11-19T05:27:01,327 INFO [M:0;f990f635ffd0:34055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=732.09 KB at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628f8d63517f4645b8dc7d7baf4a2034 2024-11-19T05:27:01,329 INFO [M:0;f990f635ffd0:34055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 628f8d63517f4645b8dc7d7baf4a2034 2024-11-19T05:27:01,342 DEBUG [M:0;f990f635ffd0:34055 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7e6619ae14743cab1c5bc9ae1a6152d is 69, key is f990f635ffd0,38449,1731993838792/rs:state/1731993841452/Put/seqid=0 2024-11-19T05:27:01,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073742454_1630 (size=5156) 2024-11-19T05:27:01,746 INFO [M:0;f990f635ffd0:34055 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2042 (bloomFilter=true), to=hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7e6619ae14743cab1c5bc9ae1a6152d 2024-11-19T05:27:01,749 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a79d3f3184de4abfae15c493b656f7ae as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a79d3f3184de4abfae15c493b656f7ae 2024-11-19T05:27:01,752 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a79d3f3184de4abfae15c493b656f7ae, entries=8, sequenceid=2042, filesize=5.5 K 2024-11-19T05:27:01,753 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628f8d63517f4645b8dc7d7baf4a2034 as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/628f8d63517f4645b8dc7d7baf4a2034 2024-11-19T05:27:01,756 INFO [M:0;f990f635ffd0:34055 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 628f8d63517f4645b8dc7d7baf4a2034 2024-11-19T05:27:01,756 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/628f8d63517f4645b8dc7d7baf4a2034, entries=162, sequenceid=2042, filesize=41.8 K 2024-11-19T05:27:01,757 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e7e6619ae14743cab1c5bc9ae1a6152d as hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7e6619ae14743cab1c5bc9ae1a6152d 2024-11-19T05:27:01,760 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45297/user/jenkins/test-data/68b23fc3-9c30-36c9-10b5-9fd973b738ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e7e6619ae14743cab1c5bc9ae1a6152d, entries=1, sequenceid=2042, filesize=5.0 K 2024-11-19T05:27:01,761 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(3140): Finished flush of dataSize ~732.64 KB/750227, heapSize ~898.13 KB/919680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=2042, compaction requested=false 2024-11-19T05:27:01,762 INFO [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T05:27:01,762 DEBUG [M:0;f990f635ffd0:34055 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731994020869Disabling compacts and flushes for region at 1731994020869Disabling writes for close at 1731994020869Obtaining lock to block concurrent updates at 1731994020869Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731994020869Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=750227, getHeapSize=919920, getOffHeapSize=0, getCellsCount=2229 at 1731994020870 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731994020870Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731994020870Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731994020889 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731994020889Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731994021303 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731994021323 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731994021323Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731994021330 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731994021342 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731994021342Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67ac2646: reopening flushed file at 1731994021749 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a2567f2: reopening flushed file at 1731994021752 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d3c21e: reopening flushed file at 1731994021756 (+4 ms)Finished flush of dataSize ~732.64 KB/750227, heapSize ~898.13 KB/919680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 891ms, sequenceid=2042, compaction requested=false at 1731994021761 (+5 ms)Writing region close event to WAL at 1731994021762 (+1 ms)Closed at 1731994021762 2024-11-19T05:27:01,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33169 is added to blk_1073741830_1006 (size=884124) 2024-11-19T05:27:01,765 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T05:27:01,765 INFO [M:0;f990f635ffd0:34055 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T05:27:01,765 INFO [M:0;f990f635ffd0:34055 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34055 2024-11-19T05:27:01,765 INFO [M:0;f990f635ffd0:34055 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T05:27:01,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T05:27:01,927 INFO [M:0;f990f635ffd0:34055 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T05:27:01,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34055-0x101518c9a1c0000, quorum=127.0.0.1:53542, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T05:27:01,971 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-19T05:27:01,976 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T05:27:01,977 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T05:27:01,977 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T05:27:01,977 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.log.dir/,STOPPED} 2024-11-19T05:27:01,979 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-19T05:27:01,979 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-19T05:27:01,979 WARN [BP-23751114-172.17.0.2-1731993833712 heartbeating to localhost/127.0.0.1:45297 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-19T05:27:01,979 WARN [BP-23751114-172.17.0.2-1731993833712 heartbeating to localhost/127.0.0.1:45297 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-23751114-172.17.0.2-1731993833712 (Datanode Uuid 101fec1c-8968-4899-8bf8-b796156e53d3) service to localhost/127.0.0.1:45297 2024-11-19T05:27:01,982 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data1/current/BP-23751114-172.17.0.2-1731993833712 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T05:27:01,983 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/cluster_6de778e3-02dd-80b1-2805-940adfd245b8/data/data2/current/BP-23751114-172.17.0.2-1731993833712 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-19T05:27:01,983 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-19T05:27:01,990 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-19T05:27:01,991 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-19T05:27:01,991 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-19T05:27:01,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-19T05:27:01,991 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/37a8456a-5c04-4cc1-8e0e-c3e76523ab90/hadoop.log.dir/,STOPPED} 2024-11-19T05:27:02,004 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T05:27:02,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down